vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Objects;
  20 import javax.inject.Inject;
  21 import javax.inject.Singleton;
  22 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  23 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  24 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  25 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  26 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  27 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  28 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  29 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  30 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  31 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  32 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  33 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  34 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  35 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  37 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  58 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 @Singleton
  63 public class VpnSubnetRouteHandler {
  64     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  65     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  66     private final DataBroker dataBroker;
  67     private final SubnetOpDpnManager subOpDpnManager;
  68     private final IBgpManager bgpManager;
  69     private final IdManagerService idManager;
  70     private final LockManagerService lockManager;
  71     private final VpnOpDataSyncer vpnOpDataSyncer;
  72     private final VpnNodeListener vpnNodeListener;
  73     private final IFibManager fibManager;
  74
  75     @Inject
  76     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  77             final IBgpManager bgpManager, final IdManagerService idManager,
  78             LockManagerService lockManagerService, final VpnOpDataSyncer vpnOpDataSyncer,
  79         final VpnNodeListener vpnNodeListener, final IFibManager fibManager) {
  80         this.dataBroker = dataBroker;
  81         this.subOpDpnManager = subnetOpDpnManager;
  82         this.bgpManager = bgpManager;
  83         this.idManager = idManager;
  84         this.lockManager = lockManagerService;
  85         this.vpnOpDataSyncer = vpnOpDataSyncer;
  86         this.vpnNodeListener = vpnNodeListener;
  87         this.fibManager = fibManager;
  88     }
  89
  90     // TODO Clean up the exception handling
  91     @SuppressWarnings("checkstyle:IllegalCatch")
  92     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  93         Uuid subnetId = subnetmap.getId();
  94         String subnetIp = subnetmap.getSubnetIp();
  95         Subnetmap subMap = null;
  96         SubnetOpDataEntry subOpEntry = null;
  97         SubnetOpDataEntryBuilder subOpBuilder = null;
  98         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
  99         Optional<SubnetOpDataEntry> optionalSubs = null;
 100
 101         Preconditions.checkNotNull(subnetId, LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 102         Preconditions.checkNotNull(subnetIp,
 103                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 104         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 105
 106         String vpnName;
 107         if (subnetmap.getVpnId() != null) {
 108             vpnName = subnetmap.getVpnId().getValue();
 109             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 110             if (vpnId == VpnConstants.INVALID_ID) {
 111                 vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 112                         VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 113                 vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 114                 if (vpnId == VpnConstants.INVALID_ID) {
 115                     LOG.error("{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 116                               + "processing subnet {} with IP {}, bailing out now.", LOGGING_PREFIX, vpnName, subnetId,
 117                             subnetIp);
 118                     return;
 119                 }
 120             }
 121         } else {
 122             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 123                       subnetId);
 124             return;
 125         }
 126         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 127                 subnetId.getValue(), subnetIp, vpnName);
 128         //TODO(vivek): Change this to use more granularized lock at subnetId level
 129         try {
 130             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 131             // Please check if subnetId belongs to an External Network
 132             InstanceIdentifier<Subnetmap> subMapid =
 133                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 134                             new SubnetmapKey(subnetId)).build();
 135             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 136                     LogicalDatastoreType.CONFIGURATION, subMapid);
 137             if (!sm.isPresent()) {
 138                 LOG.error("{} onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet {} IP {}"
 139                         + " vpnName {}", LOGGING_PREFIX, subnetId, subnetIp, vpnName);
 140                 return;
 141             }
 142             subMap = sm.get();
 143
 144             if (isBgpVpn) {
 145                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 146                         .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 147                 Optional<Networks> optionalNets = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 148                         LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 149                 if (optionalNets.isPresent()) {
 150                     LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 151                                     + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 152                             subnetId.getValue(), subnetIp, subMap.getNetworkId().getValue(), vpnName);
 153                     return;
 154                 }
 155             }
 156             //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 157             subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 158                     new SubnetOpDataEntryKey(subnetId)).build();
 159             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 160                     subOpIdentifier);
 161             if (optionalSubs.isPresent()) {
 162                 LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 163                         + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 164                 return;
 165             }
 166             LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 167                     + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 168             subOpBuilder = new SubnetOpDataEntryBuilder().withKey(new SubnetOpDataEntryKey(subnetId));
 169             subOpBuilder.setSubnetId(subnetId);
 170             subOpBuilder.setSubnetCidr(subnetIp);
 171             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 172
 173             if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 174                 LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 175                         + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 176                 return;
 177             }
 178             subOpBuilder.setVrfId(primaryRd);
 179             subOpBuilder.setVpnName(vpnName);
 180             subOpBuilder.setSubnetToDpn(new ArrayList<>());
 181             subOpBuilder.setRouteAdvState(TaskState.Idle);
 182             subOpBuilder.setElanTag(elanTag);
 183             Long l3Vni = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd).getL3vni();
 184             subOpBuilder.setL3vni(l3Vni);
 185             subOpEntry = subOpBuilder.build();
 186             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 187                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 188             LOG.info("{} onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}", LOGGING_PREFIX,
 189                     subnetId.getValue());
 190         } catch (TransactionCommitFailedException e) {
 191             LOG.error("{} Creation of SubnetOpDataEntry for subnet {} failed ", LOGGING_PREFIX,
 192                     subnetId.getValue(), e);
 193             // The second part of this method depends on subMap being non-null so fail fast here.
 194             return;
 195         } catch (RuntimeException e) { //TODO: Avoid this
 196             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 197                     subnetId.getValue(), subnetIp, vpnName, e);
 198             return;
 199         } catch (ReadFailedException e) {
 200             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 201                     subnetId, subnetIp, vpnName);
 202             return;
 203         } finally {
 204             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 205         }
 206         try {
 207             //In second critical section , Port-Op-Data will be updated.
 208             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 209             BigInteger dpnId = null;
 210             SubnetToDpn subDpn = null;
 211             Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<>();
 212
 213             optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 214                     subOpIdentifier);
 215             subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get())
 216                     .withKey(new SubnetOpDataEntryKey(subnetId));
 217             List<Uuid> portList = subMap.getPortList();
 218             if (portList != null) {
 219                 for (Uuid port : portList) {
 220                     Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 221                     if (intfState != null) {
 222                         try {
 223                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 224                         } catch (Exception e) {
 225                             LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 226                                             + " subnetroute inclusion for this interface for subnet {} subnetIp {} "
 227                                     + "vpn {} failed with exception", LOGGING_PREFIX, port.getValue(),
 228                                     subnetId.getValue(), subnetIp, vpnName, e);
 229                             continue;
 230                         }
 231                         if (dpnId.equals(BigInteger.ZERO)) {
 232                             LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 233                                             + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 234                                     subnetId.getValue(), subnetIp, vpnName);
 235                             continue;
 236                         }
 237                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 238                         if (intfState.getOperStatus() != OperStatus.Up) {
 239                             LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 240                                             + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(),
 241                                     subnetId.getValue(), subnetIp, vpnName);
 242                             continue;
 243                         }
 244                         subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 245                         if (intfState.getOperStatus() == OperStatus.Up) {
 246                             // port is UP
 247                             subDpnMap.put(dpnId, subDpn);
 248                         }
 249                     } else {
 250                         subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 251                     }
 252                 }
 253                 if (subDpnMap.size() > 0) {
 254                     subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 255                 }
 256             }
 257             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 258                     subMap.getNetworkId(), isBgpVpn);
 259             subOpEntry = subOpBuilder.build();
 260             SingleTransactionDataBroker.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 261                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 262             LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 263                             + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 264                     subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 265                     subOpEntry.getLastAdvState());
 266         } catch (RuntimeException e) {
 267             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {}", LOGGING_PREFIX,
 268                     subnetId.getValue(), subnetIp, vpnName, e);
 269         } catch (ReadFailedException e) {
 270             LOG.error("{} onSubnetAddedToVpn: Failed to read data store for subnet {} ip {} vpn {}", LOGGING_PREFIX,
 271                     subnetId, subnetIp, vpnName);
 272         } catch (TransactionCommitFailedException ex) {
 273             LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {} failed",
 274                     LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 275         } finally {
 276             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 277         }
 278     }
 279
 280     // TODO Clean up the exception handling
 281     @SuppressWarnings("checkstyle:IllegalCatch")
 282     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 283         Uuid subnetId = subnetmap.getId();
 284         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 285                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 286         //TODO(vivek): Change this to use more granularized lock at subnetId level
 287         try {
 288             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 289             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 290                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 291                             new SubnetOpDataEntryKey(subnetId)).build();
 292             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 293                     LogicalDatastoreType.OPERATIONAL,
 294                     subOpIdentifier);
 295             if (!optionalSubs.isPresent()) {
 296                 LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 297                                 + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 298                         subnetId.getValue(), subnetmap.getVpnId());
 299                 return;
 300             }
 301             LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 302                             + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 303                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 304                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 305                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 306                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 307                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 308                  * going through the list of ports on the subnet
 309                  */
 310             InstanceIdentifier<Subnetmap> subMapid =
 311                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 312                             new SubnetmapKey(subnetId)).build();
 313             Optional<Subnetmap> sm = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 314                     LogicalDatastoreType.CONFIGURATION, subMapid);
 315             if (!sm.isPresent()) {
 316                 LOG.error("{} onSubnetDeletedFromVpn: Stale ports removal: Unable to retrieve subnetmap entry"
 317                                 + " for subnet {} subnetIp {} vpnName {}", LOGGING_PREFIX, subnetId.getValue(),
 318                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName());
 319             } else {
 320                 Subnetmap subMap = sm.get();
 321                 List<Uuid> portList = subMap.getPortList();
 322                 if (portList != null) {
 323                     for (Uuid port : portList) {
 324                         InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 325                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 326                                         new PortOpDataEntryKey(port.getValue())).build();
 327                         LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 328                                         + " from subnet {} subnetIp {} vpnName {} TaskState {}",
 329                                 LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 330                                 optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 331                                 optionalSubs.get().getRouteAdvState());
 332                         SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 333                                 portOpIdentifier, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 334                     }
 335                 }
 336             }
 337
 338             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 339             String rd = subOpBuilder.getVrfId();
 340             String subnetIp = subOpBuilder.getSubnetCidr();
 341             String vpnName = subOpBuilder.getVpnName();
 342             //Withdraw the routes for all the interfaces on this subnet
 343             //Remove subnet route entry from FIB
 344             deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 345             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 346                     VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 347             LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 348                             + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(),
 349                     subnetIp, vpnName, rd);
 350         } catch (RuntimeException e) { //TODO: Avoid this
 351             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {}",
 352                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 353         } catch (TransactionCommitFailedException ex) {
 354             LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 355                             + " vpnId {} failed", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 356                     subnetmap.getVpnId(), ex);
 357         } catch (ReadFailedException e) {
 358             LOG.error("{} onSubnetDeletedFromVpn: Failed to read data store for subnet {} ip {} vpn {}",
 359                     LOGGING_PREFIX, subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 360         } finally {
 361             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 362         }
 363     }
 364
 365     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 366         Uuid subnetId = subnetmap.getId();
 367         String vpnName = subnetmap.getVpnId().getValue();
 368         String subnetIp = subnetmap.getSubnetIp();
 369
 370         Preconditions.checkNotNull(subnetId,
 371                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 372         Preconditions.checkNotNull(subnetIp,
 373                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 374         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 375         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 376         try {
 377             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 378                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 379                             new SubnetOpDataEntryKey(subnetId)).build();
 380             Optional<SubnetOpDataEntry> optionalSubs =
 381                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 382                             subOpIdentifier);
 383             if (optionalSubs.isPresent()) {
 384                 onSubnetDeletedFromVpn(subnetmap, true);
 385             } else {
 386                 onSubnetAddedToVpn(subnetmap, true, elanTag);
 387             }
 388             LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 389                     subnetId.getValue(), subnetIp, vpnName);
 390         } catch (ReadFailedException e) {
 391             LOG.error("onSubnetUpdatedInVpn: Failed to read data store for subnet{} ip {} elanTag {} vpn {}",subnetId,
 392                     subnetIp, elanTag, vpnName);
 393         }
 394     }
 395
 396     // TODO Clean up the exception handling
 397     @SuppressWarnings("checkstyle:IllegalCatch")
 398     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 399         Uuid subnetId = subnetmap.getId();
 400         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 401                 subnetId.getValue());
 402         //TODO(vivek): Change this to use more granularized lock at subnetId level
 403         try {
 404             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 405             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 406                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 407                             new SubnetOpDataEntryKey(subnetId)).build();
 408
 409             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 410                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 411             if (!optionalSubs.isPresent()) {
 412                 LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 413                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 414                 return;
 415             }
 416             String vpnName = optionalSubs.get().getVpnName();
 417             String subnetIp = optionalSubs.get().getSubnetCidr();
 418             String rd = optionalSubs.get().getVrfId();
 419             String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 420             LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 421                             + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 422                     vpnName, rd, routeAdvState);
 423             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 424             Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 425             if (intfState == null) {
 426                 // Interface State not yet available
 427                 return;
 428             }
 429             final BigInteger dpnId;
 430             try {
 431                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 432             } catch (Exception e) {
 433                 LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 434                                 + " for this interface failed for subnet {} subnetIp {} vpn {} rd {}",
 435                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd, e);
 436                 return;
 437             }
 438             if (dpnId.equals(BigInteger.ZERO)) {
 439                 LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 440                                 + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 441                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 442                 return;
 443             }
 444             subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 445             if (intfState.getOperStatus() != OperStatus.Up) {
 446                 LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 447                                 + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 448                         portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 449                 return;
 450             }
 451             LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 452                             + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 453                     subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 454             SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 455             if (subDpn == null) {
 456                 LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 457                                 + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 458                         portId.getValue(), vpnName, rd, subnetIp);
 459                 return;
 460             }
 461             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 462             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 463             List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 464             subDpnList.add(subDpn);
 465             subOpBuilder.setSubnetToDpn(subDpnList);
 466             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 467                 if (subOpBuilder.getNhDpnId() == null) {
 468                     // No nexthop selected yet, elect one now
 469                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 470                             subnetmap.getNetworkId(), true);
 471                 } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 472                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 473                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 474                 }
 475             }
 476             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 477             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 478                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 479             LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 480                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 481                     portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 482                     subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 483         } catch (RuntimeException e) { //TODO: Avoid this
 484             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {}", LOGGING_PREFIX,
 485                     portId.getValue(), subnetId.getValue(), e);
 486         } catch (ReadFailedException e) {
 487             LOG.error("{} onPortAddedToSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 488                     portId, subnetId);
 489         } catch (TransactionCommitFailedException e) {
 490             LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied",
 491                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), e);
 492         } finally {
 493             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 494         }
 495     }
 496
 497     // TODO Clean up the exception handling
 498     @SuppressWarnings("checkstyle:IllegalCatch")
 499     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 500         Uuid subnetId = subnetmap.getId();
 501         //TODO(vivek): Change this to use more granularized lock at subnetId level
 502         try {
 503             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 504             PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 505                     subnetmap.getId());
 506             if (portOpEntry == null) {
 507                 return;
 508             }
 509             BigInteger dpnId = portOpEntry.getDpnId();
 510             if (dpnId == null) {
 511                 LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 512                                 + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 513                         subnetId.getValue());
 514                 return;
 515             }
 516             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 517             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 518                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 519                             new SubnetOpDataEntryKey(subnetId)).build();
 520             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 521                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 522             if (!optionalSubs.isPresent()) {
 523                 LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 524                         + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 525                 return;
 526             }
 527             LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 528                             + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 529                     LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 530                     optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 531                     optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 532             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 533             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 534             BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 535             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 536                 // select another NhDpnId
 537                 if (last) {
 538                     LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 539                                     + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 540                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 541                     // last port on this DPN, so we need to elect the new NHDpnId
 542                     electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 543                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 544                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 545                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 546                     LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 547                                     + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 548                             portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 549                             subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 550                 }
 551             }
 552         } catch (RuntimeException e) {
 553             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {}", LOGGING_PREFIX,
 554                     portId.getValue(), subnetId.getValue(), e);
 555         } catch (ReadFailedException e) {
 556             LOG.error("{} onPortRemovedFromSubnet: Failed to read data store for port {} subnet {}", LOGGING_PREFIX,
 557                     portId, subnetId);
 558         } catch (TransactionCommitFailedException e) {
 559             LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed", LOGGING_PREFIX,
 560                     portId.getValue(), subnetId.getValue(), e);
 561         } finally {
 562             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 563         }
 564     }
 565
 566     // TODO Clean up the exception handling
 567     @SuppressWarnings("checkstyle:IllegalCatch")
 568     public void onInterfaceUp(BigInteger dpnId, String intfName, Uuid subnetId) {
 569         //TODO(vivek): Change this to use more granularized lock at subnetId level
 570         SubnetToDpn subDpn = null;
 571         if (dpnId == null || Objects.equals(dpnId, BigInteger.ZERO)) {
 572             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 573                     intfName, subnetId.getValue());
 574             return;
 575         }
 576         try {
 577             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 578             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 579                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 580                             new SubnetOpDataEntryKey(subnetId)).build();
 581             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 582                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 583             if (!optionalSubs.isPresent()) {
 584                 LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 585                         + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 586                 return;
 587             }
 588             subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 589             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 590             if (subDpn == null) {
 591                 return;
 592             }
 593             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 594             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 595             LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 596                             + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 597                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 598                     subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 599             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 600                     subnetId.getValue());
 601             List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 602             subDpnList.add(subDpn);
 603             subOpBuilder.setSubnetToDpn(subDpnList);
 604             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 605                 if (subOpBuilder.getNhDpnId() == null) {
 606                     // No nexthop selected yet, elect one now
 607                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 608                             null /*networkId*/, !isExternalSubnetVpn);
 609                 } else if (!isExternalSubnetVpn) {
 610                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 611                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 612                 }
 613             }
 614             SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 615             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 616                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 617             LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 618                             + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 619                     subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 620                     subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 621         } catch (RuntimeException e) {
 622             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {}",
 623                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 624         } catch (ReadFailedException e) {
 625             LOG.error("{} onInterfaceUp: Failed to read data store for interface {} dpn {} subnet {}", LOGGING_PREFIX,
 626                     intfName, dpnId, subnetId);
 627         } catch (TransactionCommitFailedException e) {
 628             LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed",
 629                     LOGGING_PREFIX, subnetId.getValue(), intfName, e);
 630         } finally {
 631             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 632         }
 633     }
 634
 635     // TODO Clean up the exception handling
 636     @SuppressWarnings("checkstyle:IllegalCatch")
 637     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName, Uuid subnetId) {
 638         if (dpnId == null || Objects.equals(dpnId, BigInteger.ZERO)) {
 639             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 640                     interfaceName, subnetId.getValue());
 641             return;
 642         }
 643         try {
 644             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 645             boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 646             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 647                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 648                             new SubnetOpDataEntryKey(subnetId)).build();
 649             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 650                     LogicalDatastoreType.OPERATIONAL,
 651                     subOpIdentifier);
 652             if (!optionalSubs.isPresent()) {
 653                 LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 654                         + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 655                 return;
 656             }
 657             SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 658             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 659             LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 660                             + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 661                     subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 662                     subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 663                     interfaceName);
 664             BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 665             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 666                 // select another NhDpnId
 667                 if (last) {
 668                     LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 669                                     + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 670                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 671                     // last port on this DPN, so we need to elect the new NHDpnId
 672                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 673                             !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 674                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 675                             subOpIdentifier, subOpBuilder.build(), VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 676                     LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 677                                     + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 678                             subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 679                             interfaceName);
 680                 }
 681             }
 682         } catch (RuntimeException e) { //TODO: Remove RuntimeException
 683             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {}",
 684                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 685         } catch (ReadFailedException e) {
 686             LOG.error("{} onInterfaceDown: Failed to read data store for interface {} dpn {} subnet {}",
 687                     LOGGING_PREFIX, interfaceName, dpnId, subnetId.getValue(), e);
 688         } catch (TransactionCommitFailedException ex) {
 689             LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {} failed",
 690                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 691         } finally {
 692             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 693         }
 694     }
 695
 696     // TODO Clean up the exception handling
 697     @SuppressWarnings("checkstyle:IllegalCatch")
 698     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, BigInteger dpnId) {
 699         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 700                 dpnId.toString());
 701         try {
 702             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 703             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 704                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 705                             new SubnetOpDataEntryKey(subnetId)).build();
 706             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 707                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 708             if (!optionalSubs.isPresent()) {
 709                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 710                         LOGGING_PREFIX, subnetId.getValue());
 711                 return;
 712             }
 713             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 714                             + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 715                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 716                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 717                     optionalSubs.get().getLastAdvState(), dpnId.toString());
 718             SubnetOpDataEntry subOpEntry = optionalSubs.get();
 719             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 720             boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 721             if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 722                 if (subOpBuilder.getNhDpnId() == null) {
 723                     // No nexthop selected yet, elect one now
 724                     electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 725                             null /*networkId*/, !isExternalSubnetVpn);
 726                 } else if (!isExternalSubnetVpn) {
 727                     // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 728                     getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 729                 }
 730             }
 731             subOpEntry = subOpBuilder.build();
 732             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 733                     subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 734             LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 735                             + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 736                     LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 737                     subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 738                     subOpEntry.getLastAdvState());
 739         } catch (RuntimeException e) { //TODO: lockSubnet() throws this exception. Rectify lockSubnet()
 740             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}",
 741                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 742         } catch (TransactionCommitFailedException ex) {
 743             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to update subnetRoute for subnet {} on dpn {}",
 744                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 745         } catch (ReadFailedException e) {
 746             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Failed to read data store for subnet {} on dpn {}",
 747                     LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 748         }
 749         finally {
 750             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 751         }
 752     }
 753
 754     // TODO Clean up the exception handling
 755     @SuppressWarnings("checkstyle:IllegalCatch")
 756     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, BigInteger dpnId) {
 757         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 758         //TODO(vivek): Change this to use more granularized lock at subnetId level
 759         try {
 760             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 761             InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 762                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 763                             new SubnetOpDataEntryKey(subnetId)).build();
 764             Optional<SubnetOpDataEntry> optionalSubs = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 765                     LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 766             if (!optionalSubs.isPresent()) {
 767                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 768                         + " is not available", LOGGING_PREFIX, subnetId.getValue());
 769                 return;
 770             }
 771             LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 772                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 773                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 774                     optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 775                     optionalSubs.get().getLastAdvState());
 776             SubnetOpDataEntry subOpEntry = null;
 777             SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 778             BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 779             if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 780                 electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 781                 subOpEntry = subOpBuilder.build();
 782                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 783                         subOpEntry, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
 784                 LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 785                                 + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 786                         dpnId.toString(), optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 787                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 788                         optionalSubs.get().getLastAdvState());
 789             }
 790         } catch (RuntimeException e) {
 791             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 792                     + " dpnId {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 793         } catch (ReadFailedException e) {
 794             LOG.error("{} Failed to read data store for subnet {} dpn {}", LOGGING_PREFIX, subnetId, dpnId);
 795         } catch (TransactionCommitFailedException e) {
 796             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 797                     + " on dpn {} failed", LOGGING_PREFIX, subnetId.getValue(), dpnId, e);
 798         } finally {
 799             VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 800         }
 801     }
 802
 803     @SuppressWarnings("checkstyle:IllegalCatch")
 804     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 805         try {
 806             //BGP manager will handle withdraw and advertise internally if prefix
 807             //already exist
 808             long label = 0;
 809             long l3vni = 0;
 810
 811             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 812             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 813                 l3vni = subOpBuilder.getL3vni();
 814             } else {
 815                 label = subOpBuilder.getLabel();
 816             }
 817             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 818                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 819                     0 /*l2vni*/, null /*gatewayMacAddress*/);
 820             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 821         } catch (Exception e) {
 822             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 823                     + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 824                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 825         }
 826     }
 827
 828     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 829         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 830                 subOpBuilder.getNhDpnId());
 831         if (nhTepIp != null) {
 832             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 833         } else {
 834             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 835                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 836                     subOpBuilder.getNhDpnId().toString());
 837             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 838         }
 839     }
 840
 841     // TODO Clean up the exception handling
 842     @SuppressWarnings("checkstyle:IllegalCatch")
 843     private boolean addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String nextHopIp,
 844                                         String vpnName, Long elanTag, long label, long l3vni,
 845                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 846
 847         Preconditions.checkNotNull(rd,
 848                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 849         Preconditions.checkNotNull(subnetIp,
 850                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 851         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 852         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 853         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 854         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 855         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 856         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 857                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 858         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 859                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni).setLabel(label).setElanTag(elanTag)
 860                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 861         if (!isBgpVpn) {
 862             vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 863             return true;
 864         }
 865         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 866         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, VpnUtil
 867                 .getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, vpnName), subnetIp), VpnUtil
 868                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, subnetId,
 869                         Prefixes.PrefixCue.SubnetRoute));
 870         vpnPopulator.populateFib(input, null /*writeCfgTxn*/);
 871         try {
 872             // BGP manager will handle withdraw and advertise internally if prefix
 873             // already exist
 874             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 875                     encapType, label, l3vni, 0 /*l2vni*/, null /*gatewayMacAddress*/);
 876         } catch (Exception e) {
 877             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 878                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 879             return false;
 880         }
 881         return true;
 882     }
 883
 884     private int getLabel(String rd, String subnetIp) {
 885         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 886             VpnUtil.getNextHopLabelKey(rd, subnetIp));
 887         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 888                 subnetIp);
 889         return label;
 890     }
 891
 892     // TODO Clean up the exception handling
 893     @SuppressWarnings("checkstyle:IllegalCatch")
 894     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 895         Preconditions.checkNotNull(rd,
 896                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 897         Preconditions.checkNotNull(subnetIp,
 898                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 899         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 900         if (isBgpVpn) {
 901             try {
 902                 bgpManager.withdrawPrefix(rd, subnetIp);
 903             } catch (Exception e) {
 904                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}",
 905                         LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 906                 return false;
 907             }
 908         }
 909         return true;
 910     }
 911
 912     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 913         fibManager.removeFibEntry(rd, prefix, null);
 914         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 915         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 916             String importingRd = vpnInstance.getVrfId();
 917             fibManager.removeFibEntry(importingRd, prefix, null);
 918             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 919                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 920         }
 921         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 922                 + " vpnName {}", prefix, rd, vpnName);
 923     }
 924
 925     // TODO Clean up the exception handling
 926     @SuppressWarnings("checkstyle:IllegalCatch")
 927     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, BigInteger oldDpnId, Uuid subnetId,
 928                                            Uuid networkId, boolean isBgpVpn) {
 929         List<SubnetToDpn> subDpnList = null;
 930         boolean isRouteAdvertised = false;
 931         subDpnList = subOpBuilder.getSubnetToDpn();
 932         String rd = subOpBuilder.getVrfId();
 933         String subnetIp = subOpBuilder.getSubnetCidr();
 934         String vpnName = subOpBuilder.getVpnName();
 935         long elanTag = subOpBuilder.getElanTag();
 936         boolean isAlternateDpnSelected = false;
 937         long l3vni = 0;
 938         long label = 0;
 939         String networkName = networkId != null ? networkId.getValue() : null;
 940
 941         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} TaskState {}"
 942                 + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 943                 subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 944         if (!isBgpVpn) {
 945             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 946             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 947             // blended more better into this design.
 948             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 949                 l3vni = subOpBuilder.getL3vni();
 950             } else {
 951                 label = getLabel(rd, subnetIp);
 952                 subOpBuilder.setLabel(label);
 953             }
 954             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 955                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 956             if (isRouteAdvertised) {
 957                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 958             } else {
 959                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 960                     + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 961                     vpnName, rd);
 962                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 963             }
 964             return;
 965         }
 966
 967         String nhTepIp = null;
 968         BigInteger nhDpnId = null;
 969         for (SubnetToDpn subnetToDpn : subDpnList) {
 970             if (subnetToDpn.getDpnId().equals(oldDpnId)) {
 971                 // Is this same is as input dpnId, then ignore it
 972                 continue;
 973             }
 974             nhDpnId = subnetToDpn.getDpnId();
 975             if (vpnNodeListener.isConnectedNode(nhDpnId)) {
 976                 // selected dpnId is connected to ODL
 977                 // but does it have a TEP configured at all?
 978                 try {
 979                     nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 980                     if (nhTepIp != null) {
 981                         isAlternateDpnSelected = true;
 982                         break;
 983                     }
 984                 } catch (Exception e) {
 985                     LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
 986                             + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
 987                 }
 988             }
 989         }
 990         if (!isAlternateDpnSelected) {
 991             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
 992             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
 993                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
 994                         + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName,
 995                         rd);
 996                 // Withdraw route from BGP for this subnet
 997                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 998                 subOpBuilder.setNhDpnId(null);
 999                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1000                 if (routeWithdrawn) {
1001                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
1002                 } else {
1003                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
1004                         + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
1005                         subnetIp, vpnName, rd);
1006                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
1007                 }
1008             }
1009         } else {
1010             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
1011             subOpBuilder.setNhDpnId(nhDpnId);
1012             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
1013                 l3vni = subOpBuilder.getL3vni();
1014             } else {
1015                 label = getLabel(rd, subnetIp);
1016                 subOpBuilder.setLabel(label);
1017             }
1018             //update the VRF entry for the subnetroute.
1019             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
1020                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
1021             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
1022             if (isRouteAdvertised) {
1023                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1024             } else {
1025                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1026                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1027                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1028             }
1029         }
1030     }
1031
1032     private boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1033         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1034                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1035     }
1036 }
1037