vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import java.util.*;
  11
  12 import com.google.common.base.Preconditions;
  13
  14 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  15 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  16 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  17 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  18 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  19 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  20 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  21 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  22 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.*;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.subnet.to.dpn.VpnInterfaces;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  37 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  38
  39 import java.math.BigInteger;
  40
  41 import org.slf4j.Logger;
  42 import org.slf4j.LoggerFactory;
  43
  44 import com.google.common.base.Optional;
  45
  46
  47 public class VpnSubnetRouteHandler implements NeutronvpnListener {
  48     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  49
  50     private final DataBroker broker;
  51     private SubnetOpDpnManager subOpDpnManager;
  52     private final IBgpManager bgpManager;
  53     private IdManagerService idManager;
  54     private VpnInterfaceManager vpnInterfaceManager;
  55
  56     public VpnSubnetRouteHandler(final DataBroker db, IBgpManager bgpManager, VpnInterfaceManager vpnIntfManager) {
  57         broker = db;
  58         subOpDpnManager = new SubnetOpDpnManager(broker);
  59         this.bgpManager = bgpManager;
  60         this.vpnInterfaceManager = vpnIntfManager;
  61     }
  62
  63     public void setIdManager(IdManagerService idManager) {
  64         this.idManager = idManager;
  65     }
  66
  67     @Override
  68     public void onSubnetAddedToVpn(SubnetAddedToVpn notification) {
  69         if (!notification.isExternalVpn()) {
  70             return;
  71         }
  72
  73         Uuid subnetId = notification.getSubnetId();
  74         String vpnName = notification.getVpnName();
  75         String subnetIp = notification.getSubnetIp();
  76         Long elanTag = notification.getElanTag();
  77
  78         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  79         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  80         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
  81         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  82
  83         logger.info("onSubnetAddedToVpn: Subnet " + subnetId.getValue() + " being added to vpn");
  84         //TODO(vivek): Change this to use more granularized lock at subnetId level
  85         synchronized (this) {
  86             try {
  87                 Subnetmap subMap = null;
  88
  89                 // Please check if subnetId belongs to an External Network
  90                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
  91                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
  92                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
  93                 if (!sm.isPresent()) {
  94                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
  95                     return;
  96                 }
  97                 subMap = sm.get();
  98                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class).
  99                         child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 100                 Optional<Networks> optionalNets = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 101                 if (optionalNets.isPresent()) {
 102                     logger.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network {}, so ignoring this for SubnetRoute",
 103                             subnetId.getValue(), subMap.getNetworkId().getValue());
 104                     return;
 105                 }
 106                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 107                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 108                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 109                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 110                         LogicalDatastoreType.OPERATIONAL,
 111                         subOpIdentifier);
 112                 if (optionalSubs.isPresent()) {
 113                     logger.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 114                             " already detected to be present");
 115                     return;
 116                 }
 117                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 118                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<BigInteger, SubnetToDpn>();
 119                 SubnetOpDataEntry subOpEntry = null;
 120                 BigInteger dpnId = null;
 121                 BigInteger nhDpnId = null;
 122                 SubnetToDpn subDpn = null;
 123
 124                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 125                 subOpBuilder.setSubnetId(subnetId);
 126                 subOpBuilder.setSubnetCidr(subnetIp);
 127                 String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpnName);
 128                 if (rd == null) {
 129                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
 130                     return;
 131                 }
 132                 subOpBuilder.setVrfId(rd);
 133                 subOpBuilder.setVpnName(vpnName);
 134                 subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>());
 135                 subOpBuilder.setRouteAdvState(TaskState.Na);
 136                 subOpBuilder.setElanTag(elanTag);
 137
 138                 // First recover set of ports available in this subnet
 139                 List<Uuid> portList = subMap.getPortList();
 140                 if (portList != null) {
 141                     for (Uuid port: portList) {
 142                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,port.getValue());
 143                         if (intfState != null) {
 144                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 145                             if (dpnId == null) {
 146                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
 147                                 continue;
 148                             }
 149                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 150                             if (intfState.getOperStatus() != OperStatus.Up) {
 151                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 152                                 continue;
 153                             }
 154                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 155                             if (intfState.getOperStatus() == OperStatus.Up) {
 156                                 // port is UP
 157                                 subDpnMap.put(dpnId, subDpn);
 158                                 if (nhDpnId == null) {
 159                                     nhDpnId = dpnId;
 160                                 }
 161                             }
 162                         } else {
 163                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 164                         }
 165                     }
 166                     if (subDpnMap.size() > 0) {
 167                         subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>(subDpnMap.values()));
 168                     }
 169                 }
 170
 171                 if (nhDpnId != null) {
 172                     logger.info("Next-Hop dpn {} is available for rd {} subnetIp {} vpn {}", nhDpnId, rd, subnetIp, vpnName);
 173                     subOpBuilder.setNhDpnId(nhDpnId);
 174                     try {
 175                         /*
 176                         Write the subnet route entry to the FIB.
 177                         And also advertise the subnet route entry via BGP.
 178                         */
 179                         int label = getLabel(rd, subnetIp);
 180                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 181                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 182                         subOpBuilder.setRouteAdvState(TaskState.Done);
 183                     } catch (Exception ex) {
 184                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
 185                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 186                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 187                     }
 188                 }else{
 189                     logger.info("Next-Hop dpn is unavailable for rd {} subnetIp {} vpn {}", rd, subnetIp, vpnName);
 190                 }
 191
 192                 subOpEntry = subOpBuilder.build();
 193                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 194                 logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet " + subnetId.getValue());
 195             } catch (Exception ex) {
 196                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 197                         subnetId.getValue() + " failed {}", ex);
 198             } finally {
 199             }
 200         }
 201     }
 202
 203     @Override
 204     public void onSubnetDeletedFromVpn(SubnetDeletedFromVpn notification) {
 205         Uuid subnetId = notification.getSubnetId();
 206
 207         if (!notification.isExternalVpn()) {
 208             return;
 209         }
 210         logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
 211         //TODO(vivek): Change this to use more granularized lock at subnetId level
 212         synchronized (this) {
 213             try {
 214                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 215                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 216                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 217                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 218                         LogicalDatastoreType.OPERATIONAL,
 219                         subOpIdentifier);
 220                 if (!optionalSubs.isPresent()) {
 221                     logger.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 222                             " not available in datastore");
 223                     return;
 224                 }
 225
 226                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 227                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 228                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 229                  * going through the list of ports on the subnet
 230                  */
 231                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 232                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 233                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
 234                 if (!sm.isPresent()) {
 235                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 236                 } else {
 237                     Subnetmap subMap = sm.get();
 238                     List<Uuid> portList = subMap.getPortList();
 239                     if (portList != null) {
 240                         for (Uuid port : portList) {
 241                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
 242                                     child(PortOpDataEntry.class, new PortOpDataEntryKey(port.getValue())).build();
 243                             logger.trace("Deleting portOpData entry for port " + port.getValue());
 244                             MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 245                         }
 246                     }
 247                 }
 248
 249                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 250                 String rd = subOpBuilder.getVrfId();
 251                 String subnetIp = subOpBuilder.getSubnetCidr();
 252                 String vpnName = subOpBuilder.getVpnName();
 253                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 254                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
 255                 try {
 256                     //Withdraw the routes for all the interfaces on this subnet
 257                     //Remove subnet route entry from FIB
 258                     deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 259                     withdrawSubnetRoutefromBgp(rd, subnetIp);
 260                 } catch (Exception ex) {
 261                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
 262                             subnetId.getValue() + " failed {}" + ex);
 263                 }
 264             } catch (Exception ex) {
 265                 logger.error("Removal of SubnetOpDataEntry for subnet " +
 266                         subnetId.getValue() + " failed {}" + ex);
 267             } finally {
 268             }
 269         }
 270     }
 271
 272     @Override
 273     public void onSubnetUpdatedInVpn(SubnetUpdatedInVpn notification) {
 274         Uuid subnetId = notification.getSubnetId();
 275         String vpnName = notification.getVpnName();
 276         String subnetIp = notification.getSubnetIp();
 277         Long elanTag = notification.getElanTag();
 278
 279         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 280         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 281         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 282         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 283
 284         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 285                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 286         Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 287                 LogicalDatastoreType.OPERATIONAL,
 288                 subOpIdentifier);
 289         if (optionalSubs.isPresent()) {
 290             if (!notification.isExternalVpn()) {
 291                 SubnetDeletedFromVpnBuilder bldr = new SubnetDeletedFromVpnBuilder().setVpnName(vpnName);
 292                 bldr.setElanTag(elanTag).setExternalVpn(true).setSubnetIp(subnetIp).setSubnetId(subnetId);
 293                 onSubnetDeletedFromVpn(bldr.build());
 294             }
 295             // TODO(vivek): Something got updated, but we donot know what ?
 296         } else {
 297             if (notification.isExternalVpn()) {
 298                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
 299                 bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);;
 300                 onSubnetAddedToVpn(bldr.build());
 301             }
 302             // TODO(vivek): Something got updated, but we donot know what ?
 303         }
 304     }
 305
 306     @Override
 307     public void onPortAddedToSubnet(PortAddedToSubnet notification) {
 308         Uuid subnetId = notification.getSubnetId();
 309         Uuid portId = notification.getPortId();
 310
 311         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 312         //TODO(vivek): Change this to use more granularized lock at subnetId level
 313         synchronized (this) {
 314             try {
 315                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 316                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 317
 318                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 319                         subOpIdentifier);
 320                 if (!optionalSubs.isPresent()) {
 321                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 322                             " that is not in VPN, ignoring");
 323                     return;
 324                 }
 325                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,portId.getValue());
 326                 if (intfState == null) {
 327                     // Interface State not yet available
 328                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 329                     return;
 330                 }
 331                 BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 332                 if (dpnId == null) {
 333                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 334                     return;
 335                 }
 336                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 337                 if (intfState.getOperStatus() != OperStatus.Up) {
 338                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 339                     return;
 340                 }
 341                 logger.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 342                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 343                 if (subDpn == null) {
 344                     return;
 345                 }
 346                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 347                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 348                 subDpnList.add(subDpn);
 349                 subOpBuilder.setSubnetToDpn(subDpnList);
 350                 if (subOpBuilder.getNhDpnId()  == null) {
 351                     subOpBuilder.setNhDpnId(dpnId);
 352                 }
 353                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 354                 String rd = subOpBuilder.getVrfId();
 355                 String subnetIp = subOpBuilder.getSubnetCidr();
 356                 String vpnName = subOpBuilder.getVpnName();
 357                 Long elanTag = subOpBuilder.getElanTag();
 358                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) ||
 359                         (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 360                     try {
 361                         // Write the Subnet Route Entry to FIB
 362                         // Advertise BGP Route here and set route_adv_state to DONE
 363                         int label = getLabel(rd, subnetIp);
 364                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 365                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 366                         subOpBuilder.setRouteAdvState(TaskState.Done);
 367                     } catch (Exception ex) {
 368                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
 369                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 370                     }
 371                 }
 372                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 373                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 374                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 375
 376             } catch (Exception ex) {
 377                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 378                         subnetId.getValue() + " failed {}", ex);
 379             } finally {
 380             }
 381         }
 382     }
 383
 384     @Override
 385     public void onPortRemovedFromSubnet(PortRemovedFromSubnet notification) {
 386         Uuid subnetId = notification.getSubnetId();
 387         Uuid portId = notification.getPortId();
 388
 389         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 390         //TODO(vivek): Change this to use more granularized lock at subnetId level
 391         synchronized (this) {
 392             try {
 393                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 394                 if (portOpEntry == null) {
 395                     return;
 396                 }
 397                 BigInteger dpnId = portOpEntry.getDpnId();
 398                 if (dpnId == null) {
 399                     logger.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring", portId.getValue());
 400                     return;
 401                 }
 402                 logger.debug("onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 403                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 404                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 405                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 406                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 407                         subOpIdentifier);
 408                 if (!optionalSubs.isPresent()) {
 409                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 410                             " that is not in VPN, ignoring");
 411                     return;
 412                 }
 413                 SubnetOpDataEntry subOpEntry = null;
 414                 List<SubnetToDpn> subDpnList = null;
 415                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 416                 String rd = subOpBuilder.getVrfId();
 417                 String subnetIp = subOpBuilder.getSubnetCidr();
 418                 String vpnName = subOpBuilder.getVpnName();
 419                 Long elanTag = subOpBuilder.getElanTag();
 420                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 421                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 422                     // select another NhDpnId
 423                     if (last) {
 424                         logger.debug("onPortRemovedFromSubnet: Last port " + portId + " on the subnet: " +  subnetId.getValue());
 425                         // last port on this DPN, so we need to swap the NHDpnId
 426                         subDpnList = subOpBuilder.getSubnetToDpn();
 427                         if (subDpnList.isEmpty()) {
 428                             subOpBuilder.setNhDpnId(null);
 429                             try {
 430                                 // withdraw route from BGP
 431                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 432                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 433                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 434                             } catch (Exception ex) {
 435                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 436                                   subnetId.getValue() + " from BGP failed ", ex);
 437                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 438                             }
 439                         } else {
 440                             nhDpnId = subDpnList.get(0).getDpnId();
 441                             subOpBuilder.setNhDpnId(nhDpnId);
 442                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 443                             try {
 444                                 // Best effort Withdrawal of route from BGP for this subnet
 445                                 // Advertise the new NexthopIP to BGP for this subnet
 446                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 447                                 int label = getLabel(rd, subnetIp);
 448                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 449                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 450                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 451                             } catch (Exception ex) {
 452                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
 453                                         " information for subnet " + subnetId.getValue() +
 454                                         " to BGP failed {}" + ex);
 455                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 456                             }
 457                         }
 458                     }
 459                 }
 460                 subOpEntry = subOpBuilder.build();
 461                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 462                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
 463             } catch (Exception ex) {
 464                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 465                         subnetId.getValue() + " failed {}" + ex);
 466             } finally {
 467             }
 468         }
 469     }
 470
 471     public void onInterfaceUp(Interface intfState) {
 472
 473         logger.info("onInterfaceUp: Port " + intfState.getName());
 474         //TODO(vivek): Change this to use more granularized lock at subnetId level
 475         synchronized (this) {
 476             SubnetToDpn subDpn = null;
 477             String intfName = intfState.getName();
 478             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 479             if (portOpEntry == null) {
 480                 logger.info("onInterfaceUp: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 481                 return;
 482             }
 483             BigInteger dpnId = portOpEntry.getDpnId();
 484             if (dpnId  == null) {
 485                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 486                 if (dpnId == null) {
 487                     logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
 488                     return;
 489                 }
 490             }
 491             Uuid subnetId = portOpEntry.getSubnetId();
 492             try {
 493                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 494                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 495                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 496                         subOpIdentifier);
 497                 if (!optionalSubs.isPresent()) {
 498                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 499                             " is not available");
 500                     return;
 501                 }
 502
 503                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 504                 logger.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 505                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 506                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 507                 if (subDpn == null) {
 508                     return;
 509                 }
 510                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 511                 subDpnList.add(subDpn);
 512                 subOpBuilder.setSubnetToDpn(subDpnList);
 513                 if (subOpBuilder.getNhDpnId()  == null) {
 514                     subOpBuilder.setNhDpnId(dpnId);
 515                 }
 516                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 517                 String rd = subOpBuilder.getVrfId();
 518                 String subnetIp = subOpBuilder.getSubnetCidr();
 519                 String vpnName = subOpBuilder.getVpnName();
 520                 Long elanTag = subOpBuilder.getElanTag();
 521                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) || (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 522                     try {
 523                         // Write the Subnet Route Entry to FIB
 524                         // Advertise BGP Route here and set route_adv_state to DONE
 525                         int label = getLabel(rd, subnetIp);
 526                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 527                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 528                         subOpBuilder.setRouteAdvState(TaskState.Done);
 529                     } catch (Exception ex) {
 530                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
 531                           subnetId.getValue() + " to BGP failed {}" + ex);
 532                     }
 533                 }
 534                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 535                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 536                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 537             } catch (Exception ex) {
 538                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 539                         subnetId.getValue() + " failed {}" + ex);
 540             } finally {
 541             }
 542         }
 543     }
 544
 545     public void onInterfaceDown(Interface intfState) {
 546         logger.info("onInterfaceDown: Port " + intfState.getName());
 547         //TODO(vivek): Change this to use more granularized lock at subnetId level
 548         synchronized (this) {
 549             String intfName = intfState.getName();
 550             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 551             if (portOpEntry == null) {
 552                 logger.info("onInterfaceDown: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 553                 return;
 554             }
 555             BigInteger dpnId = portOpEntry.getDpnId();
 556             if (dpnId  == null) {
 557                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 558                 if (dpnId == null) {
 559                     logger.error("onInterfaceDown: Unable to determine the DPNID for port " + intfName);
 560                     return;
 561                 }
 562             }
 563             Uuid subnetId = portOpEntry.getSubnetId();
 564             try {
 565                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 566                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, intfName);
 567                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 568                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 569                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 570                         LogicalDatastoreType.OPERATIONAL,
 571                         subOpIdentifier);
 572                 if (!optionalSubs.isPresent()) {
 573                     logger.error("onInterfaceDown: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 574                             " is not available");
 575                     return;
 576                 }
 577                 SubnetOpDataEntry subOpEntry = null;
 578                 List<SubnetToDpn> subDpnList = null;
 579                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 580                 String rd = subOpBuilder.getVrfId();
 581                 String subnetIp = subOpBuilder.getSubnetCidr();
 582                 String vpnName = subOpBuilder.getVpnName();
 583                 Long elanTag = subOpBuilder.getElanTag();
 584                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 585                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 586                     // select another NhDpnId
 587                     if (last) {
 588                         logger.debug("onInterfaceDown: Last active port " + intfState.getName() + " on the subnet: " +  subnetId.getValue());
 589                         // last port on this DPN, so we need to swap the NHDpnId
 590                         subDpnList = subOpBuilder.getSubnetToDpn();
 591                         if (subDpnList.isEmpty()) {
 592                             subOpBuilder.setNhDpnId(null);
 593                             try {
 594                                 // Withdraw route from BGP for this subnet
 595                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 596                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 597                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 598                             } catch (Exception ex) {
 599                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 600                                   subnetId.getValue() + " from BGP failed {}" + ex);
 601                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 602                             }
 603                         } else {
 604                             nhDpnId = subDpnList.get(0).getDpnId();
 605                             subOpBuilder.setNhDpnId(nhDpnId);
 606                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 607                             try {
 608                                 // Best effort Withdrawal of route from BGP for this subnet
 609                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 610                                 int label = getLabel(rd, subnetIp);
 611                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 612                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 613                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 614                             } catch (Exception ex) {
 615                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 616                                         subnetId.getValue() + " to BGP failed {}" + ex);
 617                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 618                             }
 619                         }
 620                     }
 621                 }
 622                 subOpEntry = subOpBuilder.build();
 623                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 624                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + intfName);
 625             } catch (Exception ex) {
 626                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 627                         subnetId.getValue() + " failed {}" + ex);
 628             } finally {
 629             }
 630         }
 631     }
 632
 633     @Override
 634     public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
 635     }
 636
 637     @Override
 638     public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
 639     }
 640
 641     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 642                                      Long elanTag, int label) {
 643         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 644         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 645         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 646         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 647         String nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 648         if(nexthopIp != null)
 649             vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId);
 650         else
 651             logger.info("Unable to get nextHop ip address for nextHop DPN {}. Abort adding subnet route to FIB table.", nhDpnId);
 652     }
 653
 654     private int getLabel(String rd, String subnetIp) {
 655         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 656                                         VpnUtil.getNextHopLabelKey(rd, subnetIp));
 657         logger.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
 658         return label;
 659     }
 660
 661     private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) {
 662         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 663         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 664         vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 665     }
 666
 667     private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 668                                            Long elanTag, int label) throws Exception {
 669         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 670         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 671         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 672         Preconditions.checkNotNull(nhDpnId, "nhDpnId cannot be null or empty!");
 673         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 674         String nexthopIp = null;
 675         nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 676         if (nexthopIp == null) {
 677             logger.error("createSubnetRouteInVpn: Unable to obtain endpointIp address for DPNId " + nhDpnId);
 678             throw new Exception("Unable to obtain endpointIp address for DPNId " + nhDpnId);
 679         }
 680         try {
 681             // BGPManager (inside ODL) requires a withdraw followed by advertise
 682             // due to bugs with ClusterDataChangeListener used by BGPManager.
 683             //bgpManager.withdrawPrefix(rd, subnetIp);
 684             bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
 685         } catch (Exception e) {
 686             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 687             throw e;
 688         }
 689     }
 690
 691     private void withdrawSubnetRoutefromBgp(String rd, String subnetIp) throws Exception {
 692         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 693         Preconditions.checkNotNull(subnetIp, "SubnetIp cannot be null or empty!");
 694         try {
 695             bgpManager.withdrawPrefix(rd, subnetIp);
 696         } catch (Exception e) {
 697             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 698             throw e;
 699         }
 700     }
 701 }
 702