vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.Collections;
  16 import java.util.HashMap;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Objects;
  21 import javax.inject.Inject;
  22 import javax.inject.Singleton;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  25 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  26 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  27 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  28 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  29 import org.opendaylight.netvirt.vpnmanager.VpnOpDataSyncer.VpnOpDataType;
  30 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  31 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  32 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  33 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  35 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  57 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  58 import org.slf4j.Logger;
  59 import org.slf4j.LoggerFactory;
  60
  61 @Singleton
  62 public class VpnSubnetRouteHandler {
  63     private static final Logger LOG = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  64     private static final String LOGGING_PREFIX = "SUBNETROUTE:";
  65     private final DataBroker dataBroker;
  66     private final SubnetOpDpnManager subOpDpnManager;
  67     private final IBgpManager bgpManager;
  68     private final IdManagerService idManager;
  69     private final LockManagerService lockManager;
  70     private final VpnOpDataSyncer vpnOpDataSyncer;
  71     private final VpnNodeListener vpnNodeListener;
  72     private final IFibManager fibManager;
  73
  74     @Inject
  75     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  76             final IBgpManager bgpManager, final IdManagerService idManager,
  77             LockManagerService lockManagerService, final VpnOpDataSyncer vpnOpDataSyncer,
  78         final VpnNodeListener vpnNodeListener, final IFibManager fibManager) {
  79         this.dataBroker = dataBroker;
  80         this.subOpDpnManager = subnetOpDpnManager;
  81         this.bgpManager = bgpManager;
  82         this.idManager = idManager;
  83         this.lockManager = lockManagerService;
  84         this.vpnOpDataSyncer = vpnOpDataSyncer;
  85         this.vpnNodeListener = vpnNodeListener;
  86         this.fibManager = fibManager;
  87     }
  88
  89     // TODO Clean up the exception handling
  90     @SuppressWarnings("checkstyle:IllegalCatch")
  91     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
  92         Uuid subnetId = subnetmap.getId();
  93         String subnetIp = subnetmap.getSubnetIp();
  94         Subnetmap subMap = null;
  95         SubnetOpDataEntry subOpEntry = null;
  96         SubnetOpDataEntryBuilder subOpBuilder = null;
  97         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = null;
  98         Optional<SubnetOpDataEntry> optionalSubs = null;
  99
 100         Preconditions.checkNotNull(subnetId, LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetId cannot be null or empty!");
 101         Preconditions.checkNotNull(subnetIp,
 102                 LOGGING_PREFIX + " onSubnetAddedToVpn: SubnetPrefix cannot be null or empty!");
 103         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetAddedToVpn: ElanTag cannot be null or empty!");
 104
 105         String vpnName;
 106         if (subnetmap.getVpnId() != null) {
 107             vpnName = subnetmap.getVpnId().getValue();
 108             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 109             if (vpnId == VpnConstants.INVALID_ID) {
 110                 vpnOpDataSyncer.waitForVpnDataReady(VpnOpDataType.vpnInstanceToId, vpnName,
 111                         VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS);
 112                 vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 113                 if (vpnId == VpnConstants.INVALID_ID) {
 114                     LOG.error("{} onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} "
 115                               + "processing subnet {} with IP {}, bailing out now.", LOGGING_PREFIX, vpnName, subnetId,
 116                             subnetIp);
 117                     return;
 118                 }
 119             }
 120         } else {
 121             LOG.error("onSubnetAddedToVpn: VpnId {} for subnet {} not found, bailing out", subnetmap.getVpnId(),
 122                       subnetId);
 123             return;
 124         }
 125         LOG.info("{} onSubnetAddedToVpn: Subnet {} with IP {} being added to vpn {}", LOGGING_PREFIX,
 126                 subnetId.getValue(), subnetIp, vpnName);
 127         //TODO(vivek): Change this to use more granularized lock at subnetId level
 128         try {
 129             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 130             try {
 131
 132                 // Please check if subnetId belongs to an External Network
 133                 InstanceIdentifier<Subnetmap> subMapid =
 134                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 135                         new SubnetmapKey(subnetId)).build();
 136                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 137                 if (!sm.isPresent()) {
 138                     LOG.error("{} onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet {} IP {}"
 139                             + " vpnName {}",  LOGGING_PREFIX, subnetId, subnetIp, vpnName);
 140                     return;
 141                 }
 142                 subMap = sm.get();
 143
 144                 if (isBgpVpn) {
 145                     InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class)
 146                             .child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 147                     Optional<Networks> optionalNets = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION,
 148                             netsIdentifier);
 149                     if (optionalNets.isPresent()) {
 150                         LOG.info("{} onSubnetAddedToVpn: subnet {} with IP {} is an external subnet on external "
 151                                 + "network {}, so ignoring this for SubnetRoute on vpn {}", LOGGING_PREFIX,
 152                                 subnetId.getValue(), subnetIp, subMap.getNetworkId().getValue(), vpnName);
 153                         return;
 154                     }
 155                 }
 156                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 157                 subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 158                         new SubnetOpDataEntryKey(subnetId)).build();
 159                 optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 160                 if (optionalSubs.isPresent()) {
 161                     LOG.error("{} onSubnetAddedToVpn: SubnetOpDataEntry for subnet {} with ip {} and vpn {} already"
 162                             + " detected to be present", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 163                     return;
 164                 }
 165                 LOG.debug("{} onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet {} subnetIp {} "
 166                         + "vpn {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName);
 167                 subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 168                 subOpBuilder.setSubnetId(subnetId);
 169                 subOpBuilder.setSubnetCidr(subnetIp);
 170                 String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 171
 172                 if (isBgpVpn && !VpnUtil.isBgpVpn(vpnName, primaryRd)) {
 173                     LOG.error("{} onSubnetAddedToVpn: The VPN Instance name {} does not have RD. Bailing out for"
 174                             + " subnet {} subnetIp {} ", LOGGING_PREFIX, vpnName, subnetId.getValue(), subnetIp);
 175                     return;
 176                 }
 177
 178                 subOpBuilder.setVrfId(primaryRd);
 179                 subOpBuilder.setVpnName(vpnName);
 180                 subOpBuilder.setSubnetToDpn(new ArrayList<>());
 181                 subOpBuilder.setRouteAdvState(TaskState.Idle);
 182                 subOpBuilder.setElanTag(elanTag);
 183                 Long l3Vni = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd).getL3vni();
 184                 subOpBuilder.setL3vni(l3Vni);
 185
 186                 subOpEntry = subOpBuilder.build();
 187                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 188                 LOG.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}",
 189                         subnetId.getValue());
 190             } catch (Exception ex) {
 191                 LOG.error("Creation of SubnetOpDataEntry for subnet {} failed ", subnetId.getValue(), ex);
 192                 // The second part of this method depends on subMap being non-null so fail fast here.
 193                 return;
 194             } finally {
 195                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 196             }
 197
 198             //In second critical section , Port-Op-Data will be updated.
 199             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 200             try {
 201                 BigInteger dpnId = null;
 202                 SubnetToDpn subDpn = null;
 203                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<>();
 204
 205                 optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 206                 subOpBuilder =
 207                         new SubnetOpDataEntryBuilder(optionalSubs.get()).setKey(new SubnetOpDataEntryKey(subnetId));
 208                 List<Uuid> portList = subMap.getPortList();
 209                 if (portList != null) {
 210                     for (Uuid port : portList) {
 211                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 212                         if (intfState != null) {
 213                             try {
 214                                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 215                             } catch (Exception e) {
 216                                 LOG.error("{} onSubnetAddedToVpn: Unable to obtain dpnId for interface {},"
 217                                         + " subnetroute inclusion for this interface for subnet {} subnetIp {} vpn {}"
 218                                         + " failed with exception", LOGGING_PREFIX, port.getValue(),
 219                                         subnetId.getValue(), subnetIp, vpnName, e);
 220                                 continue;
 221                             }
 222                             if (dpnId.equals(BigInteger.ZERO)) {
 223                                 LOG.error("{} onSubnetAddedToVpn: Port {} is not assigned DPN yet,"
 224                                         + " ignoring subnet {} subnetIP {} vpn {}", LOGGING_PREFIX, port.getValue(),
 225                                         subnetId.getValue(), subnetIp, vpnName);
 226                                 continue;
 227                             }
 228                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 229                             if (intfState.getOperStatus() != OperStatus.Up) {
 230                                 LOG.error("{} onSubnetAddedToVpn: Port {} is not UP yet, ignoring subnet {}"
 231                                         + " subnetIp {} vpn {}", LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 232                                         subnetIp, vpnName);
 233                                 continue;
 234                             }
 235                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 236                             if (intfState.getOperStatus() == OperStatus.Up) {
 237                                 // port is UP
 238                                 subDpnMap.put(dpnId, subDpn);
 239                             }
 240                         } else {
 241                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 242                         }
 243                     }
 244                     if (subDpnMap.size() > 0) {
 245                         subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 246                     }
 247                 }
 248                 electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 249                         subMap.getNetworkId(), isBgpVpn);
 250                 subOpEntry = subOpBuilder.build();
 251                 MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 252                 LOG.info("{} onSubnetAddedToVpn: Added PortOpDataEntry and VpnInterfaces to SubnetOpData"
 253                                 + " for subnet {} subnetIp {} vpn {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 254                         subnetId.getValue(), subnetIp, vpnName, subOpEntry.getRouteAdvState(),
 255                         subOpEntry.getLastAdvState());
 256             } catch (RuntimeException ex) {
 257                 LOG.error("{} onSubnetAddedToVpn: Creation of SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 258                         + " failed {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, ex);
 259             } finally {
 260                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 261             }
 262         } catch (RuntimeException e) {
 263             LOG.error("{} onSubnetAddedToVpn: Unable to handle subnet {} with ip {} added to vpn {} {}", LOGGING_PREFIX,
 264                     subnetId.getValue(), subnetIp, vpnName, e);
 265         }
 266     }
 267
 268     // TODO Clean up the exception handling
 269     @SuppressWarnings("checkstyle:IllegalCatch")
 270     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 271         Uuid subnetId = subnetmap.getId();
 272         LOG.info("{} onSubnetDeletedFromVpn: Subnet {} with ip {} being removed from vpnId {}", LOGGING_PREFIX,
 273                 subnetId, subnetmap.getSubnetIp(), subnetmap.getVpnId());
 274         //TODO(vivek): Change this to use more granularized lock at subnetId level
 275         try {
 276             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 277             try {
 278                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 279                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 280                         new SubnetOpDataEntryKey(subnetId)).build();
 281                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 282                         LogicalDatastoreType.OPERATIONAL,
 283                         subOpIdentifier);
 284                 if (!optionalSubs.isPresent()) {
 285                     LOG.error("{} onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet {} subnetIp {} vpn {}"
 286                             + " not available in datastore", LOGGING_PREFIX, subnetId.getValue(),
 287                             subnetId.getValue(), subnetmap.getVpnId());
 288                     return;
 289                 }
 290                 LOG.trace("{} onSubnetDeletedFromVpn: Removing the SubnetOpDataEntry node for subnet {} subnetIp {}"
 291                         + " vpnName {} rd {} TaskState {}", LOGGING_PREFIX, subnetId.getValue(),
 292                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 293                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState());
 294                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 295                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 296                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 297                  * going through the list of ports on the subnet
 298                  */
 299                 InstanceIdentifier<Subnetmap> subMapid =
 300                     InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 301                         new SubnetmapKey(subnetId)).build();
 302                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 303                 if (!sm.isPresent()) {
 304                     LOG.error("{} onSubnetDeletedFromVpn: Stale ports removal: Unable to retrieve subnetmap entry"
 305                             + " for subnet {} subnetIp {} vpnName {}", LOGGING_PREFIX, subnetId.getValue(),
 306                             optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName());
 307                 } else {
 308                     Subnetmap subMap = sm.get();
 309                     List<Uuid> portList = subMap.getPortList();
 310                     if (portList != null) {
 311                         for (Uuid port : portList) {
 312                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier =
 313                                 InstanceIdentifier.builder(PortOpData.class).child(PortOpDataEntry.class,
 314                                     new PortOpDataEntryKey(port.getValue())).build();
 315                             LOG.trace("{} onSubnetDeletedFromVpn: Deleting portOpData entry for port {}"
 316                                     + " from subnet {} subnetIp {} vpnName {} TaskState()",
 317                                     LOGGING_PREFIX, port.getValue(), subnetId.getValue(),
 318                                     optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 319                                     optionalSubs.get().getRouteAdvState());
 320                             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 321                         }
 322                     }
 323                 }
 324
 325                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 326                 String rd = subOpBuilder.getVrfId();
 327                 String subnetIp = subOpBuilder.getSubnetCidr();
 328                 String vpnName = subOpBuilder.getVpnName();
 329                 //Withdraw the routes for all the interfaces on this subnet
 330                 //Remove subnet route entry from FIB
 331                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 332                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 333                 LOG.info("{} onSubnetDeletedFromVpn: Removed subnetopdataentry successfully from Datastore"
 334                         + " for subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 335                         vpnName, rd);
 336             } catch (RuntimeException ex) {
 337                 LOG.error("{} onSubnetDeletedFromVpn: Removal of SubnetOpDataEntry for subnet {} subnetIp {}"
 338                         + " vpnId {} failed {}", LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(),
 339                         subnetmap.getVpnId(), ex);
 340             } finally {
 341                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 342             }
 343         } catch (RuntimeException e) {
 344             LOG.error("{} onSubnetDeletedFromVpn: Unable to handle subnet {} with Ip {} removed from vpn {} {}",
 345                     LOGGING_PREFIX, subnetId.getValue(), subnetmap.getSubnetIp(), subnetmap.getVpnId(), e);
 346         }
 347     }
 348
 349     public void onSubnetUpdatedInVpn(Subnetmap subnetmap, Long elanTag) {
 350         Uuid subnetId = subnetmap.getId();
 351         String vpnName = subnetmap.getVpnId().getValue();
 352         String subnetIp = subnetmap.getSubnetIp();
 353
 354         Preconditions.checkNotNull(subnetId,
 355                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetId cannot be null or empty!");
 356         Preconditions.checkNotNull(subnetIp,
 357                 LOGGING_PREFIX + " onSubnetUpdatedInVpn: SubnetPrefix cannot be null or empty!");
 358         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " onSubnetUpdatedInVpn: VpnName cannot be null or empty!");
 359         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " onSubnetUpdatedInVpn: ElanTag cannot be null or empty!");
 360
 361         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 362             InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 363                 new SubnetOpDataEntryKey(subnetId)).build();
 364         Optional<SubnetOpDataEntry> optionalSubs =
 365             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 366         if (optionalSubs.isPresent()) {
 367             onSubnetDeletedFromVpn(subnetmap, true);
 368         } else {
 369             onSubnetAddedToVpn(subnetmap, true, elanTag);
 370         }
 371         LOG.info("{} onSubnetUpdatedInVpn: subnet {} with Ip {} updated successfully for vpn {}", LOGGING_PREFIX,
 372                 subnetId.getValue(), subnetIp, vpnName);
 373     }
 374
 375     // TODO Clean up the exception handling
 376     @SuppressWarnings("checkstyle:IllegalCatch")
 377     public void onPortAddedToSubnet(Subnetmap subnetmap, Uuid portId) {
 378         Uuid subnetId = subnetmap.getId();
 379         LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {}", LOGGING_PREFIX, portId.getValue(),
 380                 subnetId.getValue());
 381         //TODO(vivek): Change this to use more granularized lock at subnetId level
 382         try {
 383             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 384             try {
 385                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 386                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 387                         new SubnetOpDataEntryKey(subnetId)).build();
 388
 389                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 390                         subOpIdentifier);
 391                 if (!optionalSubs.isPresent()) {
 392                     LOG.info("{} onPortAddedToSubnet: Port {} is part of a subnet {} that is not in VPN, ignoring",
 393                             LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 394                     return;
 395                 }
 396                 String vpnName = optionalSubs.get().getVpnName();
 397                 String subnetIp = optionalSubs.get().getSubnetCidr();
 398                 String rd = optionalSubs.get().getVrfId();
 399                 String routeAdvState = optionalSubs.get().getRouteAdvState().toString();
 400                 LOG.info("{} onPortAddedToSubnet: Port {} being added to subnet {} subnetIp {} vpnName {} rd {} "
 401                                 + "TaskState {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 402                         vpnName, rd, routeAdvState);
 403                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 404                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 405                 if (intfState == null) {
 406                     // Interface State not yet available
 407                     return;
 408                 }
 409                 final BigInteger dpnId;
 410                 try {
 411                     dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 412                 } catch (Exception e) {
 413                     LOG.error("{} onPortAddedToSubnet: Unable to obtain dpnId for interface {}. subnetroute inclusion"
 414                                     + " for this interface failed for subnet {} subnetIp {} vpn {} rd {} with "
 415                                     + "exception {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp,
 416                             vpnName, rd, e);
 417                     return;
 418                 }
 419                 if (dpnId.equals(BigInteger.ZERO)) {
 420                     LOG.error("{} onPortAddedToSubnet: Port {} is not assigned DPN yet, ignoring subnetRoute "
 421                                     + "inclusion for the interface into subnet {} subnetIp {} vpnName {} rd {}",
 422                             LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 423                     return;
 424                 }
 425                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 426                 if (intfState.getOperStatus() != OperStatus.Up) {
 427                     LOG.error("{} onPortAddedToSubnet: Port {} is not UP yet, ignoring subnetRoute inclusion for "
 428                                     + "the interface into subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 429                             portId.getValue(), subnetId.getValue(), subnetIp, vpnName, rd);
 430                     return;
 431                 }
 432                 LOG.debug("{} onPortAddedToSubnet: Port {} added. Updating the SubnetOpDataEntry node for subnet {} "
 433                                 + "subnetIp {} vpnName {} rd {} TaskState {}", LOGGING_PREFIX, portId.getValue(),
 434                         subnetId.getValue(), subnetIp, vpnName, rd, routeAdvState);
 435                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 436                 if (subDpn == null) {
 437                     LOG.error("{} onPortAddedToSubnet: subnet-to-dpn list is null for subnetId {}. portId {}, "
 438                                     + "vpnName {}, rd {}, subnetIp {}", LOGGING_PREFIX, subnetId.getValue(),
 439                             portId.getValue(), vpnName, rd, subnetIp);
 440                     return;
 441                 }
 442                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 443                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 444                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 445                 subDpnList.add(subDpn);
 446                 subOpBuilder.setSubnetToDpn(subDpnList);
 447                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 448                     if (subOpBuilder.getNhDpnId() == null) {
 449                         // No nexthop selected yet, elect one now
 450                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 451                                 subnetmap.getNetworkId(), true);
 452                     } else if (!VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue())) {
 453                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 454                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 455                     }
 456                 }
 457                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 458                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 459                 LOG.info("{} onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port {} subnet {}"
 460                         + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 461                         portId.getValue(), subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 462                         subOpBuilder.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 463             } catch (Exception ex) {
 464                 LOG.error("{} onPortAddedToSubnet: Updation of subnetOpEntry for port {} subnet {} falied {}",
 465                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), ex);
 466             } finally {
 467                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 468             }
 469         } catch (Exception e) {
 470             LOG.error("{} onPortAddedToSubnet: Unable to handle port {} added to subnet {} {}", LOGGING_PREFIX,
 471                     portId.getValue(), subnetId.getValue(), e);
 472         }
 473     }
 474
 475     // TODO Clean up the exception handling
 476     @SuppressWarnings("checkstyle:IllegalCatch")
 477     public void onPortRemovedFromSubnet(Subnetmap subnetmap, Uuid portId) {
 478         Uuid subnetId = subnetmap.getId();
 479
 480         //TODO(vivek): Change this to use more granularized lock at subnetId level
 481         try {
 482             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 483             try {
 484                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue(),
 485                                                                                subnetmap.getId());
 486                 if (portOpEntry == null) {
 487                     return;
 488                 }
 489                 BigInteger dpnId = portOpEntry.getDpnId();
 490                 if (dpnId == null) {
 491                     LOG.error("{} onPortRemovedFromSubnet:  Port {} does not have a DPNId associated,"
 492                             + " ignoring removal from subnet {}", LOGGING_PREFIX, portId.getValue(),
 493                             subnetId.getValue());
 494                     return;
 495                 }
 496                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 497                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 498                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 499                         new SubnetOpDataEntryKey(subnetId)).build();
 500                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 501                     subOpIdentifier);
 502                 if (!optionalSubs.isPresent()) {
 503                     LOG.info("{} onPortRemovedFromSubnet: Port {} is part of a subnet {} that is not in VPN,"
 504                             + " ignoring", LOGGING_PREFIX, portId.getValue(), subnetId.getValue());
 505                     return;
 506                 }
 507                 LOG.info("{} onPortRemovedFromSubnet: Port {} being removed. Updating the SubnetOpDataEntry"
 508                         + " for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}", LOGGING_PREFIX,
 509                         portId.getValue(), subnetId.getValue(), optionalSubs.get().getSubnetCidr(),
 510                         optionalSubs.get().getVpnName(), optionalSubs.get().getVrfId(),
 511                         optionalSubs.get().getRouteAdvState(), optionalSubs.get().getLastAdvState());
 512                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 513                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 514                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 515                 if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 516                     // select another NhDpnId
 517                     if (last) {
 518                         LOG.debug("{} onPortRemovedFromSubnet: Last port {} being removed from subnet {} subnetIp {}"
 519                                 + " vpnName {} rd {}", LOGGING_PREFIX, portId.getValue(), subnetId.getValue(),
 520                                 subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 521                         // last port on this DPN, so we need to elect the new NHDpnId
 522                         electNewDpnForSubnetRoute(subOpBuilder, nhDpnId, subnetId, subnetmap.getNetworkId(),
 523                                 !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 524                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 525                                 subOpBuilder.build());
 526                         LOG.info("{} onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore"
 527                                 + " removing port {} from subnet {} subnetIp {} vpnName {} rd {}", LOGGING_PREFIX,
 528                                 portId.getValue(), subnetId.getValue(), subOpBuilder.getSubnetCidr(),
 529                                 subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 530                     }
 531                 }
 532             } catch (RuntimeException ex) {
 533                 LOG.error("{} onPortRemovedFromSubnet: Removal of portOp for {} from subnet {} failed {}",
 534                         LOGGING_PREFIX, portId.getValue(), subnetId.getValue(), ex);
 535             } finally {
 536                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 537             }
 538         } catch (RuntimeException e) {
 539             LOG.error("{} onPortRemovedFromSubnet: Unable to handle port {} removed from subnet {} {}",LOGGING_PREFIX,
 540                     portId.getValue(), subnetId.getValue(), e);
 541         }
 542     }
 543
 544     // TODO Clean up the exception handling
 545     @SuppressWarnings("checkstyle:IllegalCatch")
 546     public void onInterfaceUp(BigInteger dpnId, String intfName, Uuid subnetId) {
 547         //TODO(vivek): Change this to use more granularized lock at subnetId level
 548         SubnetToDpn subDpn = null;
 549         if (dpnId == null || Objects.equals(dpnId, BigInteger.ZERO)) {
 550             LOG.error("{} onInterfaceUp: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 551                     intfName, subnetId.getValue());
 552             return;
 553         }
 554         try {
 555             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 556             try {
 557                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 558                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 559                         new SubnetOpDataEntryKey(subnetId)).build();
 560                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 561                     subOpIdentifier);
 562                 if (!optionalSubs.isPresent()) {
 563                     LOG.trace("{} onInterfaceUp: SubnetOpDataEntry for subnet {} is not available."
 564                             + " Ignoring interfaceUp for port{}", LOGGING_PREFIX, subnetId.getValue(), intfName);
 565                     return;
 566                 }
 567                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 568                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 569                 if (subDpn == null) {
 570                     return;
 571                 }
 572                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 573                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 574                 LOG.info("{} onInterfaceUp: Updating the SubnetOpDataEntry node for subnet {} subnetIp {} vpn {}"
 575                         + " rd {} TaskState {} lastTaskState {}" , LOGGING_PREFIX, subnetId.getValue(),
 576                         subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 577                         subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 578                 boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(),
 579                         subnetId.getValue());
 580                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 581                 subDpnList.add(subDpn);
 582                 subOpBuilder.setSubnetToDpn(subDpnList);
 583                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 584                     if (subOpBuilder.getNhDpnId() == null) {
 585                         // No nexthop selected yet, elect one now
 586                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 587                                 null /*networkId*/, !isExternalSubnetVpn);
 588                     } else if (!isExternalSubnetVpn) {
 589                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 590                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 591                     }
 592                 }
 593                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 594                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 595                 LOG.info("{} onInterfaceUp: Updated subnetopdataentry to OP Datastore port {} up for subnet {}"
 596                         + " subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {} ", LOGGING_PREFIX, intfName,
 597                         subnetId.getValue(), subOpEntry.getSubnetCidr(), subOpEntry.getVpnName(),
 598                         subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(), subOpEntry.getLastAdvState());
 599             } catch (Exception ex) {
 600                 LOG.error("{} onInterfaceUp: Updation of SubnetOpDataEntry for subnet {} on port {} up failed {}",
 601                         LOGGING_PREFIX, subnetId.getValue(), intfName, ex);
 602             } finally {
 603                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 604             }
 605         } catch (RuntimeException e) {
 606             LOG.error("{} onInterfaceUp: Unable to handle interface up event for port {} in subnet {} {}",
 607                     LOGGING_PREFIX, intfName, subnetId.getValue(), e);
 608         }
 609     }
 610
 611     // TODO Clean up the exception handling
 612     @SuppressWarnings("checkstyle:IllegalCatch")
 613     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName, Uuid subnetId) {
 614         if (dpnId == null || Objects.equals(dpnId, BigInteger.ZERO)) {
 615             LOG.error("{} onInterfaceDown: Unable to determine the DPNID for port {} on subnet {}", LOGGING_PREFIX,
 616                     interfaceName, subnetId.getValue());
 617             return;
 618         }
 619         try {
 620             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 621             try {
 622                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 623                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 624                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 625                         new SubnetOpDataEntryKey(subnetId)).build();
 626                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 627                     LogicalDatastoreType.OPERATIONAL,
 628                     subOpIdentifier);
 629                 if (!optionalSubs.isPresent()) {
 630                     LOG.info("{} onInterfaceDown: SubnetOpDataEntry for subnet {} is not available."
 631                             + " Ignoring port {} down event.", LOGGING_PREFIX, subnetId.getValue(), interfaceName);
 632                     return;
 633                 }
 634                 SubnetOpDataEntry subnetOpDataEntry = optionalSubs.get();
 635                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subnetOpDataEntry);
 636                 LOG.info("{} onInterfaceDown: Updating the SubnetOpDataEntry node for subnet {} subnetIp {}"
 637                         + " vpnName {} rd {} TaskState {} lastTaskState {} on port {} down", LOGGING_PREFIX,
 638                         subnetId.getValue(), subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(),
 639                         subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState(),
 640                         interfaceName);
 641                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 642                 if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 643                     // select another NhDpnId
 644                     if (last) {
 645                         LOG.debug("{} onInterfaceDown: Last active port {} on the subnet {} subnetIp {} vpn {}"
 646                                 + " rd {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(),
 647                                 subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId());
 648                         // last port on this DPN, so we need to elect the new NHDpnId
 649                         electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/,
 650                                 !VpnUtil.isExternalSubnetVpn(subnetOpDataEntry.getVpnName(), subnetId.getValue()));
 651                         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier,
 652                                 subOpBuilder.build());
 653                         LOG.info("{} onInterfaceDown: Updated subnetopdataentry for subnet {} subnetIp {} vpnName {}"
 654                                 + " rd {} to OP Datastore on port {} down ", LOGGING_PREFIX, subnetId.getValue(),
 655                                 subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(),
 656                                 interfaceName);
 657                     }
 658                 }
 659             } catch (Exception ex) {
 660                 LOG.error("{} onInterfaceDown: SubnetOpDataEntry update on interface {} down event for subnet {}"
 661                         + " falied {}", LOGGING_PREFIX, interfaceName, subnetId.getValue(), ex);
 662             } finally {
 663                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 664             }
 665         } catch (RuntimeException e) {
 666             LOG.error("{} onInterfaceDown: Unable to handle interface down event for port {} in subnet {} {}",
 667                     LOGGING_PREFIX, interfaceName, subnetId.getValue(), e);
 668         }
 669     }
 670
 671     // TODO Clean up the exception handling
 672     @SuppressWarnings("checkstyle:IllegalCatch")
 673     public void updateSubnetRouteOnTunnelUpEvent(Uuid subnetId, BigInteger dpnId) {
 674         LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 675                 dpnId.toString());
 676         try {
 677             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 678             try {
 679                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 680                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 681                         new SubnetOpDataEntryKey(subnetId)).build();
 682                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 683                     LogicalDatastoreType.OPERATIONAL,
 684                     subOpIdentifier);
 685                 if (!optionalSubs.isPresent()) {
 686                     LOG.error("{} updateSubnetRouteOnTunnelUpEvent: SubnetOpDataEntry for subnet {} is not available",
 687                             LOGGING_PREFIX, subnetId.getValue());
 688                     return;
 689                 }
 690                 LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Subnet {} subnetIp {} vpnName {} rd {} TaskState {}"
 691                         + " lastTaskState {} Dpn {}", LOGGING_PREFIX, subnetId.getValue(),
 692                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 693                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 694                         optionalSubs.get().getLastAdvState(), dpnId.toString());
 695                 SubnetOpDataEntry subOpEntry = optionalSubs.get();
 696                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(subOpEntry);
 697                 boolean isExternalSubnetVpn = VpnUtil.isExternalSubnetVpn(subOpEntry.getVpnName(), subnetId.getValue());
 698                 if (subOpBuilder.getRouteAdvState() != TaskState.Advertised) {
 699                     if (subOpBuilder.getNhDpnId() == null) {
 700                         // No nexthop selected yet, elect one now
 701                         electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId,
 702                                 null /*networkId*/, !isExternalSubnetVpn);
 703                     } else if (!isExternalSubnetVpn) {
 704                         // Already nexthop has been selected, only publishing to bgp required, so publish to bgp
 705                         getNexthopTepAndPublishRoute(subOpBuilder, subnetId);
 706                     }
 707                 }
 708                 subOpEntry = subOpBuilder.build();
 709                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 710                 LOG.info("{} updateSubnetRouteOnTunnelUpEvent: Updated subnetopdataentry to OP Datastore tunnel up"
 711                         + " on dpn {} for subnet {} subnetIp {} vpnName {} rd {} TaskState {} lastTaskState {}",
 712                         LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(), subOpEntry.getSubnetCidr(),
 713                         subOpEntry.getVpnName(), subOpEntry.getVrfId(), subOpEntry.getRouteAdvState(),
 714                         subOpEntry.getLastAdvState());
 715             } catch (RuntimeException ex) {
 716                 LOG.error("{} updateSubnetRouteOnTunnelUpEvent: updating subnetRoute for subnet {} on dpn {}",
 717                         LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), ex);
 718             } finally {
 719                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 720             }
 721         } catch (RuntimeException e) {
 722             LOG.error("{} updateSubnetRouteOnTunnelUpEvent: Unable to handle tunnel up event for subnetId {} dpnId {}"
 723                     + " with exception {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 724         }
 725     }
 726
 727     // TODO Clean up the exception handling
 728     @SuppressWarnings("checkstyle:IllegalCatch")
 729     public void updateSubnetRouteOnTunnelDownEvent(Uuid subnetId, BigInteger dpnId) {
 730         LOG.info("updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {}", subnetId.getValue(), dpnId.toString());
 731         //TODO(vivek): Change this to use more granularized lock at subnetId level
 732         try {
 733             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 734             try {
 735                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier =
 736                     InstanceIdentifier.builder(SubnetOpData.class).child(SubnetOpDataEntry.class,
 737                         new SubnetOpDataEntryKey(subnetId)).build();
 738                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 739                     LogicalDatastoreType.OPERATIONAL,
 740                     subOpIdentifier);
 741                 if (!optionalSubs.isPresent()) {
 742                     LOG.error("{} updateSubnetRouteOnTunnelDownEvent: SubnetOpDataEntry for subnet {}"
 743                             + " is not available", LOGGING_PREFIX, subnetId.getValue());
 744                     return;
 745                 }
 746                 LOG.debug("{} updateSubnetRouteOnTunnelDownEvent: Dpn {} Subnet {} subnetIp {} vpnName {} rd {}"
 747                         + " TaskState {} lastTaskState {}", LOGGING_PREFIX, dpnId.toString(), subnetId.getValue(),
 748                         optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 749                         optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 750                         optionalSubs.get().getLastAdvState());
 751                 SubnetOpDataEntry subOpEntry = null;
 752                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 753                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 754                 if (nhDpnId != null && nhDpnId.equals(dpnId)) {
 755                     electNewDpnForSubnetRoute(subOpBuilder, dpnId, subnetId, null /*networkId*/, true);
 756                     subOpEntry = subOpBuilder.build();
 757                     MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 758                     LOG.info("{} updateSubnetRouteOnTunnelDownEvent: Subnet {} Dpn {} subnetIp {} vpnName {} rd {}"
 759                             + " TaskState {} lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(),
 760                             optionalSubs.get().getSubnetCidr(), optionalSubs.get().getVpnName(),
 761                             optionalSubs.get().getVrfId(), optionalSubs.get().getRouteAdvState(),
 762                             optionalSubs.get().getLastAdvState());
 763                 }
 764             } catch (RuntimeException ex) {
 765                 LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Updation of SubnetOpDataEntry for subnet {}"
 766                         + " on dpn {} failed {}", LOGGING_PREFIX, subnetId.getValue(), dpnId, ex);
 767             } finally {
 768                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 769             }
 770         } catch (RuntimeException e) {
 771             LOG.error("{} updateSubnetRouteOnTunnelDownEvent: Unable to handle tunnel down event for subnetId {}"
 772                     + " dpnId {} with exception {}", LOGGING_PREFIX, subnetId.getValue(), dpnId.toString(), e);
 773         }
 774     }
 775
 776     @SuppressWarnings("checkstyle:IllegalCatch")
 777     private void publishSubnetRouteToBgp(SubnetOpDataEntryBuilder subOpBuilder, String nextHopIp) {
 778         try {
 779             //BGP manager will handle withdraw and advertise internally if prefix
 780             //already exist
 781             long label = 0;
 782             long l3vni = 0;
 783
 784             VrfEntry.EncapType encapType =  VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 785             if (encapType.equals(VrfEntry.EncapType.Vxlan)) {
 786                 l3vni = subOpBuilder.getL3vni();
 787             } else {
 788                 label = subOpBuilder.getLabel();
 789             }
 790             bgpManager.advertisePrefix(subOpBuilder.getVrfId(), null /*macAddress*/, subOpBuilder.getSubnetCidr(),
 791                     Arrays.asList(nextHopIp), encapType,  label, l3vni,
 792                     0 /*l2vni*/, null /*gatewayMacAddress*/);
 793             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState()).setRouteAdvState(TaskState.Advertised);
 794         } catch (Exception e) {
 795             LOG.error("{} publishSubnetRouteToBgp: Subnet route not advertised for subnet {} subnetIp {} vpn {} rd {}"
 796                     + " with dpnid {}", LOGGING_PREFIX, subOpBuilder.getSubnetId().getValue(),
 797                     subOpBuilder.getSubnetCidr(), subOpBuilder.getVpnName(), subOpBuilder.getVrfId(), nextHopIp, e);
 798         }
 799     }
 800
 801     private void getNexthopTepAndPublishRoute(SubnetOpDataEntryBuilder subOpBuilder, Uuid subnetId) {
 802         String nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker,
 803                 subOpBuilder.getNhDpnId());
 804         if (nhTepIp != null) {
 805             publishSubnetRouteToBgp(subOpBuilder, nhTepIp);
 806         } else {
 807             LOG.warn("Unable to find nexthopip for rd {} subnetroute subnetip {} for dpnid {}",
 808                     subOpBuilder.getVrfId(), subOpBuilder.getSubnetCidr(),
 809                     subOpBuilder.getNhDpnId().toString());
 810             electNewDpnForSubnetRoute(subOpBuilder, null /* oldDpnId */, subnetId, null /*networkId*/, true);
 811         }
 812     }
 813
 814     // TODO Clean up the exception handling
 815     @SuppressWarnings("checkstyle:IllegalCatch")
 816     private boolean addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String nextHopIp,
 817                                         String vpnName, Long elanTag, long label, long l3vni,
 818                                         Uuid subnetId, boolean isBgpVpn, String networkName) {
 819
 820         Preconditions.checkNotNull(rd,
 821                 LOGGING_PREFIX + " addSubnetRouteToFib: RouteDistinguisher cannot be null or empty!");
 822         Preconditions.checkNotNull(subnetIp,
 823                 LOGGING_PREFIX + " addSubnetRouteToFib: SubnetRouteIp cannot be null or empty!");
 824         Preconditions.checkNotNull(vpnName, LOGGING_PREFIX + " addSubnetRouteToFib: vpnName cannot be null or empty!");
 825         Preconditions.checkNotNull(elanTag, LOGGING_PREFIX + " addSubnetRouteToFib: elanTag cannot be null or empty!");
 826         Preconditions.checkNotNull(label, LOGGING_PREFIX + " addSubnetRouteToFib: label cannot be null or empty!");
 827         VrfEntry.EncapType encapType = VpnUtil.getEncapType(VpnUtil.isL3VpnOverVxLan(l3vni));
 828         VpnPopulator vpnPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 829         LOG.info("{} addSubnetRouteToFib: Adding SubnetRoute fib entry for vpnName {}, subnetIP {}, elanTag {}",
 830                 LOGGING_PREFIX, vpnName, subnetIp, elanTag);
 831         L3vpnInput input = new L3vpnInput().setRouteOrigin(RouteOrigin.CONNECTED).setRd(rd).setVpnName(vpnName)
 832                 .setSubnetIp(subnetIp).setNextHopIp(nextHopIp).setL3vni(l3vni).setLabel(label).setElanTag(elanTag)
 833                 .setDpnId(nhDpnId).setEncapType(encapType).setNetworkName(networkName).setPrimaryRd(rd);
 834         if (!isBgpVpn) {
 835             vpnPopulator.populateFib(input, null /*writeCfgTxn*/, null /*writeOperTxn*/);
 836             return true;
 837         }
 838         Preconditions.checkNotNull(nextHopIp, LOGGING_PREFIX + "NextHopIp cannot be null or empty!");
 839         VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, VpnUtil
 840                 .getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, vpnName), subnetIp), VpnUtil
 841                 .getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp, subnetId,
 842                         Prefixes.PrefixCue.SubnetRoute));
 843         vpnPopulator.populateFib(input, null /*writeCfgTxn*/, null /*writeOperTxn*/);
 844         try {
 845             // BGP manager will handle withdraw and advertise internally if prefix
 846             // already exist
 847             bgpManager.advertisePrefix(rd, null /*macAddress*/, subnetIp, Collections.singletonList(nextHopIp),
 848                     encapType, label, l3vni, 0 /*l2vni*/, null /*gatewayMacAddress*/);
 849         } catch (Exception e) {
 850             LOG.error("{} addSubnetRouteToFib: Subnet route not advertised for subnet {} subnetIp {} vpnName {} rd {} "
 851                     + "with dpnid {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName, rd, nhDpnId, e);
 852             return false;
 853         }
 854         return true;
 855     }
 856
 857     private int getLabel(String rd, String subnetIp) {
 858         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 859             VpnUtil.getNextHopLabelKey(rd, subnetIp));
 860         LOG.trace("{} getLabel: Allocated subnetroute label {} for rd {} prefix {}", LOGGING_PREFIX, label, rd,
 861                 subnetIp);
 862         return label;
 863     }
 864
 865     // TODO Clean up the exception handling
 866     @SuppressWarnings("checkstyle:IllegalCatch")
 867     private boolean deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName, boolean isBgpVpn) {
 868         Preconditions.checkNotNull(rd,
 869                 LOGGING_PREFIX + " deleteSubnetRouteFromFib: RouteDistinguisher cannot be null or empty!");
 870         Preconditions.checkNotNull(subnetIp,
 871                 LOGGING_PREFIX +  " deleteSubnetRouteFromFib: SubnetRouteIp cannot be null or empty!");
 872         deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 873         if (isBgpVpn) {
 874             try {
 875                 bgpManager.withdrawPrefix(rd, subnetIp);
 876             } catch (Exception e) {
 877                 LOG.error("{} deleteSubnetRouteFromFib: Subnet route not withdrawn for subnetIp {} vpn {} rd {}"
 878                         + "  due to exception {}", LOGGING_PREFIX, subnetIp, vpnName, rd, e);
 879                 return false;
 880             }
 881         }
 882         return true;
 883     }
 884
 885     public void deleteSubnetRouteFibEntryFromDS(String rd, String prefix, String vpnName) {
 886         fibManager.removeFibEntry(dataBroker, rd, prefix, null);
 887         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 888         for (VpnInstanceOpDataEntry vpnInstance : vpnsToImportRoute) {
 889             String importingRd = vpnInstance.getVrfId();
 890             fibManager.removeFibEntry(dataBroker, importingRd, prefix, null);
 891             LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Deleted imported subnet route rd {} prefix {}"
 892                     + " from vpn {} importingRd {}", rd, prefix, vpnInstance.getVpnInstanceName(), importingRd);
 893         }
 894         LOG.info("SUBNETROUTE: deleteSubnetRouteFibEntryFromDS: Removed subnetroute FIB for prefix {} rd {}"
 895                 + " vpnName {}", prefix, rd, vpnName);
 896     }
 897
 898     // TODO Clean up the exception handling
 899     @SuppressWarnings("checkstyle:IllegalCatch")
 900     private void electNewDpnForSubnetRoute(SubnetOpDataEntryBuilder subOpBuilder, BigInteger oldDpnId, Uuid subnetId,
 901                                            Uuid networkId, boolean isBgpVpn) {
 902         List<SubnetToDpn> subDpnList = null;
 903         boolean isRouteAdvertised = false;
 904         subDpnList = subOpBuilder.getSubnetToDpn();
 905         String rd = subOpBuilder.getVrfId();
 906         String subnetIp = subOpBuilder.getSubnetCidr();
 907         String vpnName = subOpBuilder.getVpnName();
 908         long elanTag = subOpBuilder.getElanTag();
 909         boolean isAlternateDpnSelected = false;
 910         long l3vni = 0;
 911         long label = 0;
 912         String networkName = networkId != null ? networkId.getValue() : null;
 913
 914         LOG.info("{} electNewDpnForSubnetRoute: Handling subnet {} subnetIp {} vpn {} rd {} TaskState {}"
 915                 + " lastTaskState {}", LOGGING_PREFIX, subnetId.getValue(), subnetIp, subOpBuilder.getVpnName(),
 916                 subOpBuilder.getVrfId(), subOpBuilder.getRouteAdvState(), subOpBuilder.getLastAdvState());
 917         if (!isBgpVpn) {
 918             // Non-BGPVPN as it stands here represents use-case of External Subnets of VLAN-Provider-Network
 919             //  TODO(Tomer):  Pulling in both external and internal VLAN-Provider-Network need to be
 920             // blended more better into this design.
 921             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 922                 l3vni = subOpBuilder.getL3vni();
 923             } else {
 924                 label = getLabel(rd, subnetIp);
 925                 subOpBuilder.setLabel(label);
 926             }
 927             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, null /* nhDpnId */, null /* nhTepIp */,
 928                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 929             if (isRouteAdvertised) {
 930                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
 931             } else {
 932                 LOG.error("{} electNewDpnForSubnetRoute: Unable to find TepIp for subnet {} subnetip {} vpnName {}"
 933                     + " rd {}, attempt next dpn", LOGGING_PREFIX, subnetId.getValue(), subnetIp,
 934                     vpnName, rd);
 935                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
 936             }
 937             return;
 938         }
 939
 940         String nhTepIp = null;
 941         BigInteger nhDpnId = null;
 942         Iterator<SubnetToDpn> subnetDpnIter = subDpnList.iterator();
 943         while (subnetDpnIter.hasNext()) {
 944             SubnetToDpn subnetToDpn = subnetDpnIter.next();
 945             if (subnetToDpn.getDpnId().equals(oldDpnId)) {
 946                 // Is this same is as input dpnId, then ignore it
 947                 continue;
 948             }
 949             nhDpnId = subnetToDpn.getDpnId();
 950             if (vpnNodeListener.isConnectedNode(nhDpnId)) {
 951                 // selected dpnId is connected to ODL
 952                 // but does it have a TEP configured at all?
 953                 try {
 954                     nhTepIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 955                     if (nhTepIp != null) {
 956                         isAlternateDpnSelected = true;
 957                         break;
 958                     }
 959                 } catch (Exception e) {
 960                     LOG.warn("{} electNewDpnForSubnetRoute: Unable to find TepIp for rd {} subnetroute subnetip {}"
 961                             + " for dpnid {}, attempt next", LOGGING_PREFIX, rd, subnetIp, nhDpnId.toString(), e);
 962                     continue;
 963                 }
 964             }
 965         }
 966         if (!isAlternateDpnSelected) {
 967             //If no alternate Dpn is selected as nextHopDpn, withdraw the subnetroute if it had a nextHop already.
 968             if (isRouteAdvertised(subOpBuilder) && oldDpnId != null) {
 969                 LOG.info("{} electNewDpnForSubnetRoute: No alternate DPN available for subnet {} subnetIp {} vpn {}"
 970                         + " rd {} Prefix withdrawn from BGP", LOGGING_PREFIX, subnetId.getValue(), subnetIp, vpnName,
 971                         rd);
 972                 // Withdraw route from BGP for this subnet
 973                 boolean routeWithdrawn = deleteSubnetRouteFromFib(rd, subnetIp, vpnName, isBgpVpn);
 974                 subOpBuilder.setNhDpnId(null);
 975                 subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
 976                 if (routeWithdrawn) {
 977                     subOpBuilder.setRouteAdvState(TaskState.Withdrawn);
 978                 } else {
 979                     LOG.error("{} electNewDpnForSubnetRoute: Withdrawing NextHopDPN {} for subnet {} subnetIp {}"
 980                         + " vpn {} rd {} from BGP failed", LOGGING_PREFIX, oldDpnId, subnetId.getValue(),
 981                         subnetIp, vpnName, rd);
 982                     subOpBuilder.setRouteAdvState(TaskState.PendingWithdraw);
 983                 }
 984             }
 985         } else {
 986             //If alternate Dpn is selected as nextHopDpn, use that for subnetroute.
 987             subOpBuilder.setNhDpnId(nhDpnId);
 988             if (VpnUtil.isL3VpnOverVxLan(subOpBuilder.getL3vni())) {
 989                 l3vni = subOpBuilder.getL3vni();
 990             } else {
 991                 label = getLabel(rd, subnetIp);
 992                 subOpBuilder.setLabel(label);
 993             }
 994             //update the VRF entry for the subnetroute.
 995             isRouteAdvertised = addSubnetRouteToFib(rd, subnetIp, nhDpnId, nhTepIp,
 996                     vpnName, elanTag, label, l3vni, subnetId, isBgpVpn, networkName);
 997             subOpBuilder.setLastAdvState(subOpBuilder.getRouteAdvState());
 998             if (isRouteAdvertised) {
 999                 subOpBuilder.setRouteAdvState(TaskState.Advertised);
1000             } else {
1001                 LOG.error("{} electNewDpnForSubnetRoute: Swapping to add new NextHopDpn {} for subnet {} subnetIp {}"
1002                         + " vpn {} rd {} failed", LOGGING_PREFIX, nhDpnId, subnetId.getValue(), subnetIp, vpnName, rd);
1003                 subOpBuilder.setRouteAdvState(TaskState.PendingAdvertise);
1004             }
1005         }
1006     }
1007
1008     private boolean isRouteAdvertised(SubnetOpDataEntryBuilder subOpBuilder) {
1009         return subOpBuilder.getRouteAdvState() == TaskState.Advertised
1010                 || subOpBuilder.getRouteAdvState() == TaskState.PendingAdvertise;
1011     }
1012 }
1013