vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  19 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  20 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  21 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  22 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  23 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  24 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  25 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnListener;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortAddedToSubnet;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortRemovedFromSubnet;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterAssociatedToVpn;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterDisassociatedFromVpn;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpn;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpnBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpn;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpnBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetUpdatedInVpn;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  53 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56
  57 public class VpnSubnetRouteHandler implements NeutronvpnListener {
  58     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  59     private final DataBroker dataBroker;
  60     private final SubnetOpDpnManager subOpDpnManager;
  61     private final IBgpManager bgpManager;
  62     private final VpnInterfaceManager vpnInterfaceManager;
  63     private final IdManagerService idManager;
  64     private LockManagerService lockManager;
  65
  66     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  67                                  final IBgpManager bgpManager, final VpnInterfaceManager vpnIntfManager,
  68                                  final IdManagerService idManager, LockManagerService lockManagerService) {
  69         this.dataBroker = dataBroker;
  70         this.subOpDpnManager = subnetOpDpnManager;
  71         this.bgpManager = bgpManager;
  72         this.vpnInterfaceManager = vpnIntfManager;
  73         this.idManager = idManager;
  74         this.lockManager = lockManagerService;
  75     }
  76
  77     @Override
  78     public void onSubnetAddedToVpn(SubnetAddedToVpn notification) {
  79         if (!notification.isExternalVpn()) {
  80             return;
  81         }
  82
  83         Uuid subnetId = notification.getSubnetId();
  84         String vpnName = notification.getVpnName();
  85         String subnetIp = notification.getSubnetIp();
  86         Long elanTag = notification.getElanTag();
  87
  88         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  89         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  90         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
  91         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  92
  93         logger.info("onSubnetAddedToVpn: Subnet " + subnetId.getValue() + " being added to vpn");
  94         //TODO(vivek): Change this to use more granularized lock at subnetId level
  95         try {
  96             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
  97             try {
  98                 Subnetmap subMap = null;
  99
 100                 // Please check if subnetId belongs to an External Network
 101                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 102                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 103                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 104                 if (!sm.isPresent()) {
 105                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 106                     return;
 107                 }
 108                 subMap = sm.get();
 109                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class).
 110                         child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 111                 Optional<Networks> optionalNets = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 112                 if (optionalNets.isPresent()) {
 113                     logger.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network {}, so ignoring this for SubnetRoute",
 114                             subnetId.getValue(), subMap.getNetworkId().getValue());
 115                     return;
 116                 }
 117                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 118                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 119                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 120                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 121                         LogicalDatastoreType.OPERATIONAL,
 122                         subOpIdentifier);
 123                 if (optionalSubs.isPresent()) {
 124                     logger.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 125                             " already detected to be present");
 126                     return;
 127                 }
 128                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 129                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<BigInteger, SubnetToDpn>();
 130                 SubnetOpDataEntry subOpEntry = null;
 131                 BigInteger dpnId = null;
 132                 BigInteger nhDpnId = null;
 133                 SubnetToDpn subDpn = null;
 134
 135                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 136                 subOpBuilder.setSubnetId(subnetId);
 137                 subOpBuilder.setSubnetCidr(subnetIp);
 138                 String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(dataBroker, vpnName);
 139                 if (rd == null) {
 140                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
 141                     return;
 142                 }
 143                 subOpBuilder.setVrfId(rd);
 144                 subOpBuilder.setVpnName(vpnName);
 145                 subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>());
 146                 subOpBuilder.setRouteAdvState(TaskState.Na);
 147                 subOpBuilder.setElanTag(elanTag);
 148
 149                 // First recover set of ports available in this subnet
 150                 List<Uuid> portList = subMap.getPortList();
 151                 if (portList != null) {
 152                     for (Uuid port: portList) {
 153                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 154                         if (intfState != null) {
 155                             try {
 156                                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 157                             } catch (Exception e) {
 158                                 logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 159                                         " subnetroute inclusion for this interface failed with exception {}",
 160                                         port.getValue(), e);
 161                                 continue;
 162                             }
 163                             if (dpnId.equals(BigInteger.ZERO)) {
 164                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
 165                                 continue;
 166                             }
 167                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 168                             if (intfState.getOperStatus() != OperStatus.Up) {
 169                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 170                                 continue;
 171                             }
 172                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 173                             if (intfState.getOperStatus() == OperStatus.Up) {
 174                                 // port is UP
 175                                 subDpnMap.put(dpnId, subDpn);
 176                                 if (nhDpnId == null) {
 177                                     nhDpnId = dpnId;
 178                                 }
 179                             }
 180                         } else {
 181                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 182                         }
 183                     }
 184                     if (subDpnMap.size() > 0) {
 185                         subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>(subDpnMap.values()));
 186                     }
 187                 }
 188
 189                 if (nhDpnId != null) {
 190                     logger.info("Next-Hop dpn {} is available for rd {} subnetIp {} vpn {}", nhDpnId, rd, subnetIp, vpnName);
 191                     subOpBuilder.setNhDpnId(nhDpnId);
 192                     try {
 193                         /*
 194                         Write the subnet route entry to the FIB.
 195                         And also advertise the subnet route entry via BGP.
 196                         */
 197                         int label = getLabel(rd, subnetIp);
 198                         if (label == 0) {
 199                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of subnet {} to vpn {}", subnetIp, vpnName);
 200                             return;
 201                         }
 202                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 203                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 204                         subOpBuilder.setRouteAdvState(TaskState.Done);
 205                     } catch (Exception ex) {
 206                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
 207                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 208                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 209                     }
 210                 }else{
 211                     logger.info("Next-Hop dpn is unavailable for rd {} subnetIp {} vpn {}", rd, subnetIp, vpnName);
 212                 }
 213
 214                 subOpEntry = subOpBuilder.build();
 215                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 216                 logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}",
 217                         subnetId.getValue());
 218             } catch (Exception ex) {
 219                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 220                         subnetId.getValue() + " failed {}", ex);
 221             } finally {
 222                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 223             }
 224         } catch (Exception e) {
 225             logger.error("Unable to handle subnet {} added to vpn {} {}", subnetIp, vpnName, e);
 226         }
 227     }
 228
 229     @Override
 230     public void onSubnetDeletedFromVpn(SubnetDeletedFromVpn notification) {
 231         Uuid subnetId = notification.getSubnetId();
 232
 233         if (!notification.isExternalVpn()) {
 234             return;
 235         }
 236         logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
 237         //TODO(vivek): Change this to use more granularized lock at subnetId level
 238         try {
 239             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 240             try {
 241                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 242                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 243                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 244                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 245                         LogicalDatastoreType.OPERATIONAL,
 246                         subOpIdentifier);
 247                 if (!optionalSubs.isPresent()) {
 248                     logger.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 249                             " not available in datastore");
 250                     return;
 251                 }
 252
 253                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 254                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 255                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 256                  * going through the list of ports on the subnet
 257                  */
 258                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 259                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 260                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 261                 if (!sm.isPresent()) {
 262                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 263                 } else {
 264                     Subnetmap subMap = sm.get();
 265                     List<Uuid> portList = subMap.getPortList();
 266                     if (portList != null) {
 267                         for (Uuid port : portList) {
 268                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
 269                                     child(PortOpDataEntry.class, new PortOpDataEntryKey(port.getValue())).build();
 270                             logger.trace("Deleting portOpData entry for port " + port.getValue());
 271                             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 272                         }
 273                     }
 274                 }
 275
 276                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 277                 String rd = subOpBuilder.getVrfId();
 278                 String subnetIp = subOpBuilder.getSubnetCidr();
 279                 String vpnName = subOpBuilder.getVpnName();
 280                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 281                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
 282                 try {
 283                     //Withdraw the routes for all the interfaces on this subnet
 284                     //Remove subnet route entry from FIB
 285                     deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 286                     withdrawSubnetRoutefromBgp(rd, subnetIp);
 287                 } catch (Exception ex) {
 288                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
 289                             subnetId.getValue() + " failed {}" + ex);
 290                 }
 291             } catch (Exception ex) {
 292                 logger.error("Removal of SubnetOpDataEntry for subnet " +
 293                         subnetId.getValue() + " failed {}" + ex);
 294             } finally {
 295                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 296             }
 297         } catch (Exception e) {
 298             logger.error("Unable to handle subnet {} removed to vpn {} {}", notification.getSubnetIp(), notification.getVpnName(), e);
 299         }
 300     }
 301
 302     @Override
 303     public void onSubnetUpdatedInVpn(SubnetUpdatedInVpn notification) {
 304         Uuid subnetId = notification.getSubnetId();
 305         String vpnName = notification.getVpnName();
 306         String subnetIp = notification.getSubnetIp();
 307         Long elanTag = notification.getElanTag();
 308
 309         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 310         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 311         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 312         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 313
 314         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 315                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 316         Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 317                 LogicalDatastoreType.OPERATIONAL,
 318                 subOpIdentifier);
 319         if (optionalSubs.isPresent()) {
 320             if (!notification.isExternalVpn()) {
 321                 SubnetDeletedFromVpnBuilder bldr = new SubnetDeletedFromVpnBuilder().setVpnName(vpnName);
 322                 bldr.setElanTag(elanTag).setExternalVpn(true).setSubnetIp(subnetIp).setSubnetId(subnetId);
 323                 onSubnetDeletedFromVpn(bldr.build());
 324             }
 325             // TODO(vivek): Something got updated, but we donot know what ?
 326         } else {
 327             if (notification.isExternalVpn()) {
 328                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
 329                 bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);;
 330                 onSubnetAddedToVpn(bldr.build());
 331             }
 332             // TODO(vivek): Something got updated, but we donot know what ?
 333         }
 334     }
 335
 336     @Override
 337     public void onPortAddedToSubnet(PortAddedToSubnet notification) {
 338         Uuid subnetId = notification.getSubnetId();
 339         Uuid portId = notification.getPortId();
 340
 341         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 342         //TODO(vivek): Change this to use more granularized lock at subnetId level
 343         try {
 344             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 345             try {
 346                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 347                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 348
 349                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 350                         subOpIdentifier);
 351                 if (!optionalSubs.isPresent()) {
 352                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 353                             " that is not in VPN, ignoring");
 354                     return;
 355                 }
 356                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 357                 if (intfState == null) {
 358                     // Interface State not yet available
 359                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 360                     return;
 361                 }
 362                 BigInteger dpnId = BigInteger.ZERO;
 363                 try {
 364                     dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 365                 } catch (Exception e) {
 366                     logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 367                             " subnetroute inclusion for this interface failed with exception {}",
 368                             portId.getValue(), e);
 369                     return;
 370                 }
 371                 if (dpnId.equals(BigInteger.ZERO)) {
 372                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 373                     return;
 374                 }
 375                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 376                 if (intfState.getOperStatus() != OperStatus.Up) {
 377                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 378                     return;
 379                 }
 380                 logger.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 381                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 382                 if (subDpn == null) {
 383                     return;
 384                 }
 385                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 386                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 387                 subDpnList.add(subDpn);
 388                 subOpBuilder.setSubnetToDpn(subDpnList);
 389                 if (subOpBuilder.getNhDpnId()  == null) {
 390                     subOpBuilder.setNhDpnId(dpnId);
 391                 }
 392                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 393                 String rd = subOpBuilder.getVrfId();
 394                 String subnetIp = subOpBuilder.getSubnetCidr();
 395                 String vpnName = subOpBuilder.getVpnName();
 396                 Long elanTag = subOpBuilder.getElanTag();
 397                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) ||
 398                         (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 399                     try {
 400                         // Write the Subnet Route Entry to FIB
 401                         // Advertise BGP Route here and set route_adv_state to DONE
 402                         int label = getLabel(rd, subnetIp);
 403                         if (label == 0) {
 404                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of port {} to subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
 405                             return;
 406                         }
 407                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 408                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 409                         subOpBuilder.setRouteAdvState(TaskState.Done);
 410                     } catch (Exception ex) {
 411                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
 412                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 413                     }
 414                 }
 415                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 416                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 417                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 418
 419             } catch (Exception ex) {
 420                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 421                         subnetId.getValue() + " failed {}", ex);
 422             } finally {
 423                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 424             }
 425         } catch (Exception e) {
 426             logger.error("Unable to handle port {} added to subnet {} {}", portId.getValue(), subnetId.getValue(), e);
 427         }
 428     }
 429
 430     @Override
 431     public void onPortRemovedFromSubnet(PortRemovedFromSubnet notification) {
 432         Uuid subnetId = notification.getSubnetId();
 433         Uuid portId = notification.getPortId();
 434
 435         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 436         //TODO(vivek): Change this to use more granularized lock at subnetId level
 437         try {
 438             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 439             try {
 440                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 441                 if (portOpEntry == null) {
 442                     return;
 443                 }
 444                 BigInteger dpnId = portOpEntry.getDpnId();
 445                 if (dpnId == null) {
 446                     logger.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring", portId.getValue());
 447                     return;
 448                 }
 449                 logger.debug("onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 450                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 451                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 452                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 453                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 454                         subOpIdentifier);
 455                 if (!optionalSubs.isPresent()) {
 456                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 457                             " that is not in VPN, ignoring");
 458                     return;
 459                 }
 460                 SubnetOpDataEntry subOpEntry = null;
 461                 List<SubnetToDpn> subDpnList = null;
 462                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 463                 String rd = subOpBuilder.getVrfId();
 464                 String subnetIp = subOpBuilder.getSubnetCidr();
 465                 String vpnName = subOpBuilder.getVpnName();
 466                 Long elanTag = subOpBuilder.getElanTag();
 467                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 468                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 469                     // select another NhDpnId
 470                     if (last) {
 471                         logger.debug("onPortRemovedFromSubnet: Last port " + portId + " on the subnet: " +  subnetId.getValue());
 472                         // last port on this DPN, so we need to swap the NHDpnId
 473                         subDpnList = subOpBuilder.getSubnetToDpn();
 474                         if (subDpnList.isEmpty()) {
 475                             subOpBuilder.setNhDpnId(null);
 476                             try {
 477                                 // withdraw route from BGP
 478                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 479                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 480                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 481                             } catch (Exception ex) {
 482                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 483                                   subnetId.getValue() + " from BGP failed ", ex);
 484                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 485                             }
 486                         } else {
 487                             nhDpnId = subDpnList.get(0).getDpnId();
 488                             subOpBuilder.setNhDpnId(nhDpnId);
 489                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 490                             try {
 491                                 // Best effort Withdrawal of route from BGP for this subnet
 492                                 // Advertise the new NexthopIP to BGP for this subnet
 493                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 494                                 int label = getLabel(rd, subnetIp);
 495                                 if (label == 0) {
 496                                     logger.error("Unable to fetch label from Id Manager. Bailing out of handling removal  of port {} from subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
 497                                     return;
 498                                 }
 499                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 500                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 501                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 502                             } catch (Exception ex) {
 503                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
 504                                         " information for subnet " + subnetId.getValue() +
 505                                         " to BGP failed {}" + ex);
 506                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 507                             }
 508                         }
 509                     }
 510                 }
 511                 subOpEntry = subOpBuilder.build();
 512                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 513                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
 514             } catch (Exception ex) {
 515                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 516                         subnetId.getValue() + " failed {}" + ex);
 517             } finally {
 518                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 519             }
 520         } catch (Exception e) {
 521             logger.error("Unable to handle port {} removed from subnet {} {}", portId.getValue(), subnetId.getValue(), e);
 522         }
 523     }
 524
 525     public void onInterfaceUp(BigInteger dpnId, String intfName) {
 526         logger.info("onInterfaceUp: Port " + intfName);
 527         //TODO(vivek): Change this to use more granularized lock at subnetId level
 528         SubnetToDpn subDpn = null;
 529         PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 530         if (portOpEntry == null) {
 531             logger.info("onInterfaceUp: Port " + intfName  + "is part of a subnet not in VPN, ignoring");
 532             return;
 533         }
 534         if ((dpnId == null) || (dpnId == BigInteger.ZERO)) {
 535             dpnId = portOpEntry.getDpnId();
 536             if (dpnId == null) {
 537                 logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
 538             return;
 539             }
 540         }
 541         Uuid subnetId = portOpEntry.getSubnetId();
 542         try {
 543             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 544             try {
 545                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 546                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 547                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 548                         subOpIdentifier);
 549                 if (!optionalSubs.isPresent()) {
 550                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 551                             " is not available");
 552                     return;
 553                 }
 554
 555                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 556                 logger.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 557                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 558                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 559                 if (subDpn == null) {
 560                     return;
 561                 }
 562                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 563                 subDpnList.add(subDpn);
 564                 subOpBuilder.setSubnetToDpn(subDpnList);
 565                 if (subOpBuilder.getNhDpnId()  == null) {
 566                     subOpBuilder.setNhDpnId(dpnId);
 567                 }
 568                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 569                 String rd = subOpBuilder.getVrfId();
 570                 String subnetIp = subOpBuilder.getSubnetCidr();
 571                 String vpnName = subOpBuilder.getVpnName();
 572                 Long elanTag = subOpBuilder.getElanTag();
 573                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) || (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 574                     try {
 575                         // Write the Subnet Route Entry to FIB
 576                         // Advertise BGP Route here and set route_adv_state to DONE
 577                         int label = getLabel(rd, subnetIp);
 578                         if (label == 0) {
 579                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface up event for port {} for subnet {} in vpn {}", intfName, subnetIp, vpnName);
 580                             return;
 581                         }
 582                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 583                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 584                         subOpBuilder.setRouteAdvState(TaskState.Done);
 585                     } catch (Exception ex) {
 586                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
 587                                 subnetId.getValue() + " to BGP failed {}" + ex);
 588                     }
 589                 }
 590                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 591                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 592                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 593             } catch (Exception ex) {
 594                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 595                         subnetId.getValue() + " failed {}" + ex);
 596             } finally {
 597                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 598             }
 599         } catch (Exception e) {
 600             logger.error("Unable to handle interface up event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
 601         }
 602     }
 603
 604     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName) {
 605         logger.info("onInterfaceDown: Port " + interfaceName);
 606         //TODO(vivek): Change this to use more granularized lock at subnetId level
 607         PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(interfaceName);
 608         if (portOpEntry == null) {
 609             logger.info("onInterfaceDown: Port " + interfaceName  + "is part of a subnet not in VPN, ignoring");
 610             return;
 611         }
 612         if ((dpnId  == null) ||(dpnId == BigInteger.ZERO)) {
 613             logger.error("onInterfaceDown: Unable to determine the DPNID for port " + interfaceName);
 614             return;
 615         }
 616         Uuid subnetId = portOpEntry.getSubnetId();
 617         try {
 618             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 619             try {
 620                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 621                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 622                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 623                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 624                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 625                         LogicalDatastoreType.OPERATIONAL,
 626                         subOpIdentifier);
 627                 if (!optionalSubs.isPresent()) {
 628                     logger.error("onInterfaceDown: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 629                             " is not available");
 630                     return;
 631                 }
 632                 SubnetOpDataEntry subOpEntry = null;
 633                 List<SubnetToDpn> subDpnList = null;
 634                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 635                 String rd = subOpBuilder.getVrfId();
 636                 String subnetIp = subOpBuilder.getSubnetCidr();
 637                 String vpnName = subOpBuilder.getVpnName();
 638                 Long elanTag = subOpBuilder.getElanTag();
 639                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 640                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 641                     // select another NhDpnId
 642                     if (last) {
 643                         logger.debug("onInterfaceDown: Last active port " + interfaceName + " on the subnet: " +  subnetId.getValue());
 644                         // last port on this DPN, so we need to swap the NHDpnId
 645                         subDpnList = subOpBuilder.getSubnetToDpn();
 646                         if (subDpnList.isEmpty()) {
 647                             subOpBuilder.setNhDpnId(null);
 648                             try {
 649                                 // Withdraw route from BGP for this subnet
 650                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 651                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 652                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 653                             } catch (Exception ex) {
 654                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 655                                   subnetId.getValue() + " from BGP failed {}" + ex);
 656                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 657                             }
 658                         } else {
 659                             nhDpnId = subDpnList.get(0).getDpnId();
 660                             subOpBuilder.setNhDpnId(nhDpnId);
 661                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 662                             try {
 663                                 // Best effort Withdrawal of route from BGP for this subnet
 664                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 665                                 int label = getLabel(rd, subnetIp);
 666                                 if (label == 0) {
 667                                     logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface down event for port {} in subnet {} for vpn {}", interfaceName, subnetIp, vpnName);
 668                                     return;
 669                                 }
 670                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 671                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag, label);
 672                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 673                             } catch (Exception ex) {
 674                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 675                                         subnetId.getValue() + " to BGP failed {}" + ex);
 676                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 677                             }
 678                         }
 679                     }
 680                 }
 681                 subOpEntry = subOpBuilder.build();
 682                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 683                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + interfaceName);
 684             } catch (Exception ex) {
 685                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 686                         subnetId.getValue() + " failed {}" + ex);
 687             } finally {
 688                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 689             }
 690         } catch (Exception e) {
 691             logger.error("Unable to handle interface down event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
 692         }
 693     }
 694
 695     @Override
 696     public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
 697     }
 698
 699     @Override
 700     public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
 701     }
 702
 703     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 704                                      Long elanTag, int label) {
 705         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 706         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 707         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 708         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 709         String nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 710         if(nexthopIp != null)
 711             vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId , null);
 712         else
 713             logger.info("Unable to get nextHop ip address for nextHop DPN {}. Abort adding subnet route to FIB table.", nhDpnId);
 714     }
 715
 716     private int getLabel(String rd, String subnetIp) {
 717         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 718                                         VpnUtil.getNextHopLabelKey(rd, subnetIp));
 719         logger.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
 720         return label;
 721     }
 722
 723     private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) {
 724         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 725         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 726         vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 727     }
 728
 729     private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 730                                            Long elanTag, int label) throws Exception {
 731         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 732         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 733         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 734         Preconditions.checkNotNull(nhDpnId, "nhDpnId cannot be null or empty!");
 735         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 736         String nexthopIp = null;
 737         nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 738         if (nexthopIp == null) {
 739             logger.error("createSubnetRouteInVpn: Unable to obtain endpointIp address for DPNId " + nhDpnId);
 740             throw new Exception("Unable to obtain endpointIp address for DPNId " + nhDpnId);
 741         }
 742         try {
 743             // BGPManager (inside ODL) requires a withdraw followed by advertise
 744             // due to bugs with ClusterDataChangeListener used by BGPManager.
 745             //bgpManager.withdrawPrefix(rd, subnetIp);
 746             bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
 747         } catch (Exception e) {
 748             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 749             throw e;
 750         }
 751     }
 752
 753     private void withdrawSubnetRoutefromBgp(String rd, String subnetIp) throws Exception {
 754         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 755         Preconditions.checkNotNull(subnetIp, "SubnetIp cannot be null or empty!");
 756         try {
 757             bgpManager.withdrawPrefix(rd, subnetIp);
 758         } catch (Exception e) {
 759             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 760             throw e;
 761         }
 762     }
 763 }
 764