vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import java.util.ArrayList;
  11 import java.util.List;
  12 import java.util.Map;
  13 import java.util.HashMap;
  14
  15 import com.google.common.base.Preconditions;
  16
  17 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  18 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  19 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  20 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  21 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  22 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  23 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.*;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.*;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.subnet.to.dpn.VpnInterfaces;
  36 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  40 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  41
  42 import java.math.BigInteger;
  43
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import com.google.common.base.Optional;
  48
  49
  50 public class VpnSubnetRouteHandler implements NeutronvpnListener {
  51     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  52
  53     private final DataBroker broker;
  54     private SubnetOpDpnManager subOpDpnManager;
  55     private final IBgpManager bgpManager;
  56     private IdManagerService idManager;
  57     private VpnInterfaceManager vpnInterfaceManager;
  58
  59     public VpnSubnetRouteHandler(final DataBroker db, IBgpManager bgpManager, VpnInterfaceManager vpnIntfManager) {
  60         broker = db;
  61         subOpDpnManager = new SubnetOpDpnManager(broker);
  62         this.bgpManager = bgpManager;
  63         this.vpnInterfaceManager = vpnIntfManager;
  64     }
  65
  66     public void setIdManager(IdManagerService idManager) {
  67         this.idManager = idManager;
  68     }
  69
  70     @Override
  71     public void onSubnetAddedToVpn(SubnetAddedToVpn notification) {
  72         if (!notification.isExternalVpn()) {
  73             return;
  74         }
  75
  76         Uuid subnetId = notification.getSubnetId();
  77         String vpnName = notification.getVpnName();
  78         String subnetIp = notification.getSubnetIp();
  79         Long elanTag = notification.getElanTag();
  80
  81         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  82         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  83         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
  84         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  85
  86         logger.info("onSubnetAddedToVpn: Subnet " + subnetId.getValue() + " being added to vpn");
  87         //TODO(vivek): Change this to use more granularized lock at subnetId level
  88         synchronized (this) {
  89             try {
  90                 Subnetmap subMap = null;
  91
  92                 // Please check if subnetId belongs to an External Network
  93                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
  94                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
  95                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
  96                 if (!sm.isPresent()) {
  97                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
  98                     return;
  99                 }
 100                 subMap = sm.get();
 101                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class).
 102                         child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 103                 Optional<Networks> optionalNets = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 104                 if (optionalNets.isPresent()) {
 105                     logger.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network {}, so ignoring this for SubnetRoute",
 106                             subnetId.getValue(), subMap.getNetworkId().getValue());
 107                     return;
 108                 }
 109                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 110                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 111                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 112                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 113                         LogicalDatastoreType.OPERATIONAL,
 114                         subOpIdentifier);
 115                 if (optionalSubs.isPresent()) {
 116                     logger.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 117                             " already detected to be present");
 118                     return;
 119                 }
 120                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 121                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<>();
 122                 SubnetOpDataEntry subOpEntry = null;
 123                 BigInteger dpnId = null;
 124                 BigInteger nhDpnId = null;
 125                 SubnetToDpn subDpn = null;
 126
 127                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 128                 subOpBuilder.setSubnetId(subnetId);
 129                 subOpBuilder.setSubnetCidr(subnetIp);
 130                 String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpnName);
 131                 if (rd == null) {
 132                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
 133                     return;
 134                 }
 135                 subOpBuilder.setVrfId(rd);
 136                 subOpBuilder.setVpnName(vpnName);
 137                 subOpBuilder.setSubnetToDpn(new ArrayList<>());
 138                 subOpBuilder.setRouteAdvState(TaskState.Na);
 139                 subOpBuilder.setElanTag(elanTag);
 140
 141                 // First recover set of ports available in this subnet
 142                 List<Uuid> portList = subMap.getPortList();
 143                 if (portList != null) {
 144                     for (Uuid port: portList) {
 145                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,port.getValue());
 146                         if (intfState != null) {
 147                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 148                             if (dpnId == null) {
 149                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
 150                                 continue;
 151                             }
 152                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 153                             if (intfState.getOperStatus() != OperStatus.Up) {
 154                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 155                                 continue;
 156                             }
 157                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 158                             if (intfState.getOperStatus() == OperStatus.Up) {
 159                                 // port is UP
 160                                 subDpnMap.put(dpnId, subDpn);
 161                                 if (nhDpnId == null) {
 162                                     nhDpnId = dpnId;
 163                                 }
 164                             }
 165                         } else {
 166                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 167                         }
 168                     }
 169                     if (subDpnMap.size() > 0) {
 170                         subOpBuilder.setSubnetToDpn(new ArrayList<>(subDpnMap.values()));
 171                     }
 172                 }
 173
 174                 if (nhDpnId != null) {
 175                     subOpBuilder.setNhDpnId(nhDpnId);
 176                     try {
 177                         /*
 178                         Write the subnet route entry to the FIB.
 179                         And also advertise the subnet route entry via BGP.
 180                         */
 181                         int label = getLabel(rd, subnetIp);
 182                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 183                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 184                         subOpBuilder.setRouteAdvState(TaskState.Done);
 185                     } catch (Exception ex) {
 186                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
 187                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 188                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 189                     }
 190                 } else {
 191                     try {
 192                         /*
 193                         Write the subnet route entry to the FIB.
 194                         NOTE: Will not advertise to BGP as NextHopDPN is not available yet.
 195                         */
 196                         int label = getLabel(rd, subnetIp);
 197                         addSubnetRouteToFib(rd, subnetIp, null, vpnName, elanTag, label);
 198                     } catch (Exception ex) {
 199                         logger.error("onSubnetAddedToVpn: FIB rules writing for subnet {} with exception {} " +
 200                                 subnetId.getValue(), ex);
 201                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 202                     }
 203                 }
 204
 205                 subOpEntry = subOpBuilder.build();
 206                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 207                 logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet " + subnetId.getValue());
 208             } catch (Exception ex) {
 209                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 210                         subnetId.getValue() + " failed {}", ex);
 211             } finally {
 212             }
 213         }
 214     }
 215
 216     @Override
 217     public void onSubnetDeletedFromVpn(SubnetDeletedFromVpn notification) {
 218         Uuid subnetId = notification.getSubnetId();
 219
 220         if (!notification.isExternalVpn()) {
 221             return;
 222         }
 223         logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed to vpn");
 224         //TODO(vivek): Change this to use more granularized lock at subnetId level
 225         synchronized (this) {
 226             try {
 227                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 228                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 229                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 230                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 231                         LogicalDatastoreType.OPERATIONAL,
 232                         subOpIdentifier);
 233                 if (!optionalSubs.isPresent()) {
 234                     logger.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 235                             " not available in datastore");
 236                     return;
 237                 }
 238                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 239                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 240                 for (SubnetToDpn subDpn: subDpnList) {
 241                     List<VpnInterfaces> vpnIntfList = subDpn.getVpnInterfaces();
 242                     for (VpnInterfaces vpnIntf: vpnIntfList) {
 243                         subOpDpnManager.removePortOpDataEntry(vpnIntf.getInterfaceName());
 244                     }
 245                 }
 246                 //Removing Stale Ports in portOpData
 247                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 248                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 249                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
 250                 if (!sm.isPresent()) {
 251                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 252                 }
 253                 Subnetmap subMap = sm.get();
 254                 List<Uuid> portList = subMap.getPortList();
 255                 if(portList!=null){
 256                     InstanceIdentifier<PortOpData> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).build();
 257                     Optional<PortOpData> optionalPortOp = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 258                     if(!optionalPortOp.isPresent()){
 259                         logger.error("Stale ports removal: Cannot delete port. Not available in data store");
 260                         return;
 261                     } else{
 262                         PortOpData portOpData = optionalPortOp.get();
 263                         List<PortOpDataEntry> portOpDataList = portOpData.getPortOpDataEntry();
 264                         if(portOpDataList!=null){
 265                             for(PortOpDataEntry portOpDataListEntry :  portOpDataList){
 266                                 if(portList.contains(new Uuid(portOpDataListEntry.getPortId()))){
 267                                     logger.trace("Removing stale port: " + portOpDataListEntry + "for dissociated subnetId: " + subnetId);
 268                                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier.
 269                                             child(PortOpDataEntry.class, new PortOpDataEntryKey(portOpDataListEntry.getKey())));
 270                                 }
 271                             }
 272                         }
 273                     }
 274                 }
 275
 276                 String rd = subOpBuilder.getVrfId();
 277                 String subnetIp = subOpBuilder.getSubnetCidr();
 278                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 279                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 280                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
 281                 try {
 282                     //Withdraw the routes for all the interfaces on this subnet
 283                     //Remove subnet route entry from FIB
 284                     deleteSubnetRouteFromFib(rd, subnetIp);
 285                     withdrawSubnetRoutefromBgp(rd, subnetIp);
 286                 } catch (Exception ex) {
 287                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
 288                             subnetId.getValue() + " failed {}" + ex);
 289                 }
 290             } catch (Exception ex) {
 291                 logger.error("Removal of SubnetOpDataEntry for subnet " +
 292                         subnetId.getValue() + " failed {}" + ex);
 293             } finally {
 294             }
 295         }
 296     }
 297
 298     @Override
 299     public void onSubnetUpdatedInVpn(SubnetUpdatedInVpn notification) {
 300         Uuid subnetId = notification.getSubnetId();
 301         String vpnName = notification.getVpnName();
 302         String subnetIp = notification.getSubnetIp();
 303         Long elanTag = notification.getElanTag();
 304
 305         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 306         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 307         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 308         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 309
 310         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 311                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 312         Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 313                 LogicalDatastoreType.OPERATIONAL,
 314                 subOpIdentifier);
 315         if (optionalSubs.isPresent()) {
 316             if (!notification.isExternalVpn()) {
 317                 SubnetDeletedFromVpnBuilder bldr = new SubnetDeletedFromVpnBuilder().setVpnName(vpnName);
 318                 bldr.setElanTag(elanTag).setExternalVpn(true).setSubnetIp(subnetIp).setSubnetId(subnetId);
 319                 onSubnetDeletedFromVpn(bldr.build());
 320             }
 321             // TODO(vivek): Something got updated, but we donot know what ?
 322         } else {
 323             if (notification.isExternalVpn()) {
 324                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
 325                 bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);
 326                 onSubnetAddedToVpn(bldr.build());
 327             }
 328             // TODO(vivek): Something got updated, but we donot know what ?
 329         }
 330     }
 331
 332     @Override
 333     public void onPortAddedToSubnet(PortAddedToSubnet notification) {
 334         Uuid subnetId = notification.getSubnetId();
 335         Uuid portId = notification.getPortId();
 336
 337         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 338         //TODO(vivek): Change this to use more granularized lock at subnetId level
 339         synchronized (this) {
 340             try {
 341                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 342                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 343
 344                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 345                         subOpIdentifier);
 346                 if (!optionalSubs.isPresent()) {
 347                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 348                             " that is not in VPN, ignoring");
 349                     return;
 350                 }
 351                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,portId.getValue());
 352                 if (intfState == null) {
 353                     // Interface State not yet available
 354                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 355                     return;
 356                 }
 357                 BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 358                 if (dpnId == null) {
 359                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 360                     return;
 361                 }
 362                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 363                 if (intfState.getOperStatus() != OperStatus.Up) {
 364                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 365                     return;
 366                 }
 367                 logger.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 368                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 369                 if (subDpn == null) {
 370                     return;
 371                 }
 372                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 373                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 374                 subDpnList.add(subDpn);
 375                 subOpBuilder.setSubnetToDpn(subDpnList);
 376                 if (subOpBuilder.getNhDpnId()  == null) {
 377                     subOpBuilder.setNhDpnId(dpnId);
 378                 }
 379                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 380                 String rd = subOpBuilder.getVrfId();
 381                 String subnetIp = subOpBuilder.getSubnetCidr();
 382                 String vpnName = subOpBuilder.getVpnName();
 383                 Long elanTag = subOpBuilder.getElanTag();
 384                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) ||
 385                         (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 386                     try {
 387                         // Write the Subnet Route Entry to FIB
 388                         // Advertise BGP Route here and set route_adv_state to DONE
 389                         int label = getLabel(rd, subnetIp);
 390                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 391                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 392                         subOpBuilder.setRouteAdvState(TaskState.Done);
 393                     } catch (Exception ex) {
 394                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
 395                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 396                     }
 397                 }
 398                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 399                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 400                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 401
 402             } catch (Exception ex) {
 403                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 404                         subnetId.getValue() + " failed {}", ex);
 405             } finally {
 406             }
 407         }
 408     }
 409
 410     @Override
 411     public void onPortRemovedFromSubnet(PortRemovedFromSubnet notification) {
 412         Uuid subnetId = notification.getSubnetId();
 413         Uuid portId = notification.getPortId();
 414
 415         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 416         //TODO(vivek): Change this to use more granularized lock at subnetId level
 417         synchronized (this) {
 418             try {
 419                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 420                 if (portOpEntry == null) {
 421                     return;
 422                 }
 423                 BigInteger dpnId = portOpEntry.getDpnId();
 424                 if (dpnId == null) {
 425                     logger.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring", portId.getValue());
 426                     return;
 427                 }
 428                 logger.debug("onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 429                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 430                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 431                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 432                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 433                         subOpIdentifier);
 434                 if (!optionalSubs.isPresent()) {
 435                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 436                             " that is not in VPN, ignoring");
 437                     return;
 438                 }
 439                 SubnetOpDataEntry subOpEntry = null;
 440                 List<SubnetToDpn> subDpnList = null;
 441                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 442                 String rd = subOpBuilder.getVrfId();
 443                 String subnetIp = subOpBuilder.getSubnetCidr();
 444                 String vpnName = subOpBuilder.getVpnName();
 445                 Long elanTag = subOpBuilder.getElanTag();
 446                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 447                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 448                     // select another NhDpnId
 449                     if (last) {
 450                         logger.debug("onPortRemovedFromSubnet: Last port " + portId + " on the subnet: " +  subnetId.getValue());
 451                         // last port on this DPN, so we need to swap the NHDpnId
 452                         subDpnList = subOpBuilder.getSubnetToDpn();
 453                         if (subDpnList.isEmpty()) {
 454                             subOpBuilder.setNhDpnId(null);
 455                             try {
 456                                 // withdraw route from BGP
 457                                 deleteSubnetRouteFromFib(rd, subnetIp);
 458                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 459                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 460                             } catch (Exception ex) {
 461                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 462                                         subnetId.getValue() + " from BGP failed ", ex);
 463                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 464                             }
 465                         } else {
 466                             nhDpnId = subDpnList.get(0).getDpnId();
 467                             subOpBuilder.setNhDpnId(nhDpnId);
 468                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 469                             try {
 470                                 // Best effort Withdrawal of route from BGP for this subnet
 471                                 // Advertise the new NexthopIP to BGP for this subnet
 472                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 473                                 int label = getLabel(rd, subnetIp);
 474                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 475                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 476                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 477                             } catch (Exception ex) {
 478                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
 479                                         " information for subnet " + subnetId.getValue() +
 480                                         " to BGP failed {}" + ex);
 481                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 482                             }
 483                         }
 484                     }
 485                 }
 486                 subOpEntry = subOpBuilder.build();
 487                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 488                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
 489             } catch (Exception ex) {
 490                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 491                         subnetId.getValue() + " failed {}" + ex);
 492             } finally {
 493             }
 494         }
 495     }
 496
 497     public void onInterfaceUp(Interface intfState) {
 498
 499         logger.info("onInterfaceUp: Port " + intfState.getName());
 500         //TODO(vivek): Change this to use more granularized lock at subnetId level
 501         synchronized (this) {
 502             SubnetToDpn subDpn = null;
 503             String intfName = intfState.getName();
 504             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 505             if (portOpEntry == null) {
 506                 logger.info("onInterfaceUp: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 507                 return;
 508             }
 509             BigInteger dpnId = portOpEntry.getDpnId();
 510             if (dpnId  == null) {
 511                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 512                 if (dpnId == null) {
 513                     logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
 514                     return;
 515                 }
 516             }
 517             Uuid subnetId = portOpEntry.getSubnetId();
 518             try {
 519                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 520                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 521                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 522                         subOpIdentifier);
 523                 if (!optionalSubs.isPresent()) {
 524                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 525                             " is not available");
 526                     return;
 527                 }
 528
 529                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 530                 logger.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 531                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 532                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 533                 if (subDpn == null) {
 534                     return;
 535                 }
 536                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 537                 subDpnList.add(subDpn);
 538                 subOpBuilder.setSubnetToDpn(subDpnList);
 539                 if (subOpBuilder.getNhDpnId()  == null) {
 540                     subOpBuilder.setNhDpnId(dpnId);
 541                 }
 542                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 543                 String rd = subOpBuilder.getVrfId();
 544                 String subnetIp = subOpBuilder.getSubnetCidr();
 545                 String vpnName = subOpBuilder.getVpnName();
 546                 Long elanTag = subOpBuilder.getElanTag();
 547                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) || (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 548                     try {
 549                         // Write the Subnet Route Entry to FIB
 550                         // Advertise BGP Route here and set route_adv_state to DONE
 551                         int label = getLabel(rd, subnetIp);
 552                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 553                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 554                         subOpBuilder.setRouteAdvState(TaskState.Done);
 555                     } catch (Exception ex) {
 556                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
 557                                 subnetId.getValue() + " to BGP failed {}" + ex);
 558                     }
 559                 }
 560                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 561                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 562                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 563             } catch (Exception ex) {
 564                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 565                         subnetId.getValue() + " failed {}" + ex);
 566             } finally {
 567             }
 568         }
 569     }
 570
 571     public void onInterfaceDown(Interface intfState) {
 572         logger.info("onInterfaceDown: Port " + intfState.getName());
 573         //TODO(vivek): Change this to use more granularized lock at subnetId level
 574         synchronized (this) {
 575             String intfName = intfState.getName();
 576             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 577             if (portOpEntry == null) {
 578                 logger.info("onInterfaceDown: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 579                 return;
 580             }
 581             BigInteger dpnId = portOpEntry.getDpnId();
 582             if (dpnId  == null) {
 583                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 584                 if (dpnId == null) {
 585                     logger.error("onInterfaceDown: Unable to determine the DPNID for port " + intfName);
 586                     return;
 587                 }
 588             }
 589             Uuid subnetId = portOpEntry.getSubnetId();
 590             try {
 591                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 592                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, intfName);
 593                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 594                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 595                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 596                         LogicalDatastoreType.OPERATIONAL,
 597                         subOpIdentifier);
 598                 if (!optionalSubs.isPresent()) {
 599                     logger.error("onInterfaceDown: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 600                             " is not available");
 601                     return;
 602                 }
 603                 SubnetOpDataEntry subOpEntry = null;
 604                 List<SubnetToDpn> subDpnList = null;
 605                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 606                 String rd = subOpBuilder.getVrfId();
 607                 String subnetIp = subOpBuilder.getSubnetCidr();
 608                 String vpnName = subOpBuilder.getVpnName();
 609                 Long elanTag = subOpBuilder.getElanTag();
 610                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 611                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 612                     // select another NhDpnId
 613                     if (last) {
 614                         logger.debug("onInterfaceDown: Last active port " + intfState.getName() + " on the subnet: " +  subnetId.getValue());
 615                         // last port on this DPN, so we need to swap the NHDpnId
 616                         subDpnList = subOpBuilder.getSubnetToDpn();
 617                         if (subDpnList.isEmpty()) {
 618                             subOpBuilder.setNhDpnId(null);
 619                             try {
 620                                 // Withdraw route from BGP for this subnet
 621                                 deleteSubnetRouteFromFib(rd, subnetIp);
 622                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 623                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 624                             } catch (Exception ex) {
 625                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 626                                         subnetId.getValue() + " from BGP failed {}" + ex);
 627                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 628                             }
 629                         } else {
 630                             nhDpnId = subDpnList.get(0).getDpnId();
 631                             subOpBuilder.setNhDpnId(nhDpnId);
 632                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 633                             try {
 634                                 // Best effort Withdrawal of route from BGP for this subnet
 635                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 636                                 int label = getLabel(rd, subnetIp);
 637                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 638                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 639                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 640                             } catch (Exception ex) {
 641                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 642                                         subnetId.getValue() + " to BGP failed {}" + ex);
 643                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 644                             }
 645                         }
 646                     }
 647                 }
 648                 subOpEntry = subOpBuilder.build();
 649                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 650                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + intfName);
 651             } catch (Exception ex) {
 652                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 653                         subnetId.getValue() + " failed {}" + ex);
 654             } finally {
 655             }
 656         }
 657     }
 658
 659     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 660                                      Long elanTag, int label) {
 661         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 662         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 663         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 664         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 665         String nexthopIp = null;
 666         if (nhDpnId != null) {
 667             nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 668         }
 669         vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, subnetIp, nexthopIp, label, elanTag);
 670     }
 671
 672     private int getLabel(String rd, String subnetIp) {
 673         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 674                 VpnUtil.getNextHopLabelKey(rd, subnetIp));
 675         logger.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
 676         return label;
 677     }
 678
 679     private void deleteSubnetRouteFromFib(String rd, String subnetIp) {
 680         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 681         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 682         vpnInterfaceManager.removeFibEntryFromDS(rd, subnetIp);
 683     }
 684
 685     private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 686                                            Long elanTag, int label) throws Exception {
 687         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 688         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 689         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 690         Preconditions.checkNotNull(nhDpnId, "nhDpnId cannot be null or empty!");
 691         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 692         String nexthopIp = null;
 693         nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 694         if (nexthopIp == null) {
 695             logger.error("createSubnetRouteInVpn: Unable to obtain endpointIp address for DPNId " + nhDpnId);
 696             throw new Exception("Unable to obtain endpointIp address for DPNId " + nhDpnId);
 697         }
 698         try {
 699             // BGPManager (inside ODL) requires a withdraw followed by advertise
 700             // due to bugs with ClusterDataChangeListener used by BGPManager.
 701             bgpManager.withdrawPrefix(rd, subnetIp);
 702             bgpManager.advertisePrefix(rd, subnetIp, nexthopIp, label);
 703         } catch (Exception e) {
 704             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 705             throw e;
 706         }
 707     }
 708
 709     private void withdrawSubnetRoutefromBgp(String rd, String subnetIp) throws Exception {
 710         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 711         Preconditions.checkNotNull(subnetIp, "SubnetIp cannot be null or empty!");
 712         try {
 713             bgpManager.withdrawPrefix(rd, subnetIp);
 714         } catch (Exception e) {
 715             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 716             throw e;
 717         }
 718     }
 719
 720     @Override
 721     public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
 722     }
 723
 724     @Override
 725     public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
 726
 727     }
 728 }
 729