vpnservice/vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Arrays;
  15 import java.util.HashMap;
  16 import java.util.List;
  17 import java.util.Map;
  18 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  19 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  20 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  21 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  22 import org.opendaylight.netvirt.vpnmanager.utilities.InterfaceUtils;
  23 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  24 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  25 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.PortOpData;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetOpData;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.TaskState;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExternalNetworks;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.Networks;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.networks.NetworksKey;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NeutronvpnListener;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortAddedToSubnet;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.PortRemovedFromSubnet;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterAssociatedToVpn;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.RouterDisassociatedFromVpn;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpn;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetAddedToVpnBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpn;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetDeletedFromVpnBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.SubnetUpdatedInVpn;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  53 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  54 import org.slf4j.Logger;
  55 import org.slf4j.LoggerFactory;
  56
  57 public class VpnSubnetRouteHandler implements NeutronvpnListener {
  58     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  59     private final DataBroker dataBroker;
  60     private final SubnetOpDpnManager subOpDpnManager;
  61     private final IBgpManager bgpManager;
  62     private final VpnInterfaceManager vpnInterfaceManager;
  63     private final IdManagerService idManager;
  64     private LockManagerService lockManager;
  65
  66     public VpnSubnetRouteHandler(final DataBroker dataBroker, final SubnetOpDpnManager subnetOpDpnManager,
  67                                  final IBgpManager bgpManager, final VpnInterfaceManager vpnIntfManager,
  68                                  final IdManagerService idManager, LockManagerService lockManagerService) {
  69         this.dataBroker = dataBroker;
  70         this.subOpDpnManager = subnetOpDpnManager;
  71         this.bgpManager = bgpManager;
  72         this.vpnInterfaceManager = vpnIntfManager;
  73         this.idManager = idManager;
  74         this.lockManager = lockManagerService;
  75     }
  76
  77     @Override
  78     public void onSubnetAddedToVpn(SubnetAddedToVpn notification) {
  79         if (!notification.isExternalVpn()) {
  80             return;
  81         }
  82
  83         Uuid subnetId = notification.getSubnetId();
  84         String vpnName = notification.getVpnName();
  85         String subnetIp = notification.getSubnetIp();
  86         Long elanTag = notification.getElanTag();
  87
  88         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  89         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  90         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
  91         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  92
  93         logger.info("onSubnetAddedToVpn: Subnet " + subnetId.getValue() + " being added to vpn");
  94         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
  95         if (vpnId == VpnConstants.INVALID_ID) {
  96             vpnInterfaceManager.waitForVpnInstance(vpnName, VpnConstants.PER_VPN_INSTANCE_MAX_WAIT_TIME_IN_MILLISECONDS, vpnInterfaceManager.getvpnInstanceToIdSynchronizerMap());
  97             vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
  98             if (vpnId == VpnConstants.INVALID_ID) {
  99                 logger.error("onSubnetAddedToVpn: VpnInstance to VPNId mapping not yet available for VpnName {} processing subnet {} with IP {} " +
 100                         ", bailing out now.", vpnName, subnetId, subnetIp);
 101                 return;
 102             }
 103         }
 104         //TODO(vivek): Change this to use more granularized lock at subnetId level
 105         try {
 106             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 107             try {
 108                 Subnetmap subMap = null;
 109
 110                 // Please check if subnetId belongs to an External Network
 111                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 112                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 113                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 114                 if (!sm.isPresent()) {
 115                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 116                     return;
 117                 }
 118                 subMap = sm.get();
 119                 InstanceIdentifier<Networks> netsIdentifier = InstanceIdentifier.builder(ExternalNetworks.class).
 120                         child(Networks.class, new NetworksKey(subMap.getNetworkId())).build();
 121                 Optional<Networks> optionalNets = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, netsIdentifier);
 122                 if (optionalNets.isPresent()) {
 123                     logger.info("onSubnetAddedToVpn: subnet {} is an external subnet on external network {}, so ignoring this for SubnetRoute",
 124                             subnetId.getValue(), subMap.getNetworkId().getValue());
 125                     return;
 126                 }
 127                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
 128                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 129                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 130                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 131                         LogicalDatastoreType.OPERATIONAL,
 132                         subOpIdentifier);
 133                 if (optionalSubs.isPresent()) {
 134                     logger.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 135                             " already detected to be present");
 136                     return;
 137                 }
 138                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 139                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<BigInteger, SubnetToDpn>();
 140                 SubnetOpDataEntry subOpEntry = null;
 141                 BigInteger dpnId = null;
 142                 BigInteger nhDpnId = null;
 143                 SubnetToDpn subDpn = null;
 144
 145                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 146                 subOpBuilder.setSubnetId(subnetId);
 147                 subOpBuilder.setSubnetCidr(subnetIp);
 148                 String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(dataBroker, vpnName);
 149                 if (rd == null) {
 150                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
 151                     return;
 152                 }
 153                 subOpBuilder.setVrfId(rd);
 154                 subOpBuilder.setVpnName(vpnName);
 155                 subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>());
 156                 subOpBuilder.setRouteAdvState(TaskState.Na);
 157                 subOpBuilder.setElanTag(elanTag);
 158
 159                 // First recover set of ports available in this subnet
 160                 List<Uuid> portList = subMap.getPortList();
 161                 if (portList != null) {
 162                     for (Uuid port: portList) {
 163                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,port.getValue());
 164                         if (intfState != null) {
 165                             try {
 166                                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 167                             } catch (Exception e) {
 168                                 logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 169                                         " subnetroute inclusion for this interface failed with exception {}",
 170                                         port.getValue(), e);
 171                                 continue;
 172                             }
 173                             if (dpnId.equals(BigInteger.ZERO)) {
 174                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
 175                                 continue;
 176                             }
 177                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 178                             if (intfState.getOperStatus() != OperStatus.Up) {
 179                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 180                                 continue;
 181                             }
 182                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 183                             if (intfState.getOperStatus() == OperStatus.Up) {
 184                                 // port is UP
 185                                 subDpnMap.put(dpnId, subDpn);
 186                                 if (nhDpnId == null) {
 187                                     nhDpnId = dpnId;
 188                                 }
 189                             }
 190                         } else {
 191                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 192                         }
 193                     }
 194                     if (subDpnMap.size() > 0) {
 195                         subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>(subDpnMap.values()));
 196                     }
 197                 }
 198
 199                 if (nhDpnId != null) {
 200                     logger.info("Next-Hop dpn {} is available for rd {} subnetIp {} vpn {}", nhDpnId, rd, subnetIp, vpnName);
 201                     subOpBuilder.setNhDpnId(nhDpnId);
 202                     try {
 203                         /*
 204                         Write the subnet route entry to the FIB.
 205                         And also advertise the subnet route entry via BGP.
 206                         */
 207                         int label = getLabel(rd, subnetIp);
 208                         if (label == 0) {
 209                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of subnet {} to vpn {}", subnetIp, vpnName);
 210                             return;
 211                         }
 212                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
 213                         subOpBuilder.setRouteAdvState(TaskState.Done);
 214                     } catch (Exception ex) {
 215                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
 216                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 217                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 218                     }
 219                 }else{
 220                     logger.info("Next-Hop dpn is unavailable for rd {} subnetIp {} vpn {}", rd, subnetIp, vpnName);
 221                 }
 222
 223                 subOpEntry = subOpBuilder.build();
 224                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 225                 logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet {}",
 226                         subnetId.getValue());
 227             } catch (Exception ex) {
 228                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 229                         subnetId.getValue() + " failed {}", ex);
 230             } finally {
 231                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 232             }
 233         } catch (Exception e) {
 234             logger.error("Unable to handle subnet {} added to vpn {} {}", subnetIp, vpnName, e);
 235         }
 236
 237     }
 238
 239     @Override
 240     public void onSubnetDeletedFromVpn(SubnetDeletedFromVpn notification) {
 241         Uuid subnetId = notification.getSubnetId();
 242
 243         if (!notification.isExternalVpn()) {
 244             return;
 245         }
 246         logger.info("onSubnetDeletedFromVpn: Subnet " + subnetId.getValue() + " being removed from vpn");
 247         //TODO(vivek): Change this to use more granularized lock at subnetId level
 248         try {
 249             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 250             try {
 251                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 252                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 253                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 254                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 255                         LogicalDatastoreType.OPERATIONAL,
 256                         subOpIdentifier);
 257                 if (!optionalSubs.isPresent()) {
 258                     logger.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 259                             " not available in datastore");
 260                     return;
 261                 }
 262
 263                 /* If subnet is deleted (or if its removed from VPN), the ports that are DOWN on that subnet
 264                  * will continue to be stale in portOpData DS, as subDpnList used for portOpData removal will
 265                  * contain only ports that are UP. So here we explicitly cleanup the ports of the subnet by
 266                  * going through the list of ports on the subnet
 267                  */
 268                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 269                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 270                 Optional<Subnetmap> sm = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, subMapid);
 271                 if (!sm.isPresent()) {
 272                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 273                 } else {
 274                     Subnetmap subMap = sm.get();
 275                     List<Uuid> portList = subMap.getPortList();
 276                     if (portList != null) {
 277                         for (Uuid port : portList) {
 278                             InstanceIdentifier<PortOpDataEntry> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).
 279                                     child(PortOpDataEntry.class, new PortOpDataEntryKey(port.getValue())).build();
 280                             logger.trace("Deleting portOpData entry for port " + port.getValue());
 281                             MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 282                         }
 283                     }
 284                 }
 285
 286                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 287                 String rd = subOpBuilder.getVrfId();
 288                 String subnetIp = subOpBuilder.getSubnetCidr();
 289                 String vpnName = subOpBuilder.getVpnName();
 290                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 291                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 292                 logger.info("onSubnetDeletedFromVpn: Removed subnetopdataentry for subnet {} successfully from Datastore", subnetId.getValue());
 293                 try {
 294                     //Withdraw the routes for all the interfaces on this subnet
 295                     //Remove subnet route entry from FIB
 296                     deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 297                 } catch (Exception ex) {
 298                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
 299                             subnetId.getValue() + " failed {}" + ex);
 300                 }
 301             } catch (Exception ex) {
 302                 logger.error("Removal of SubnetOpDataEntry for subnet " +
 303                         subnetId.getValue() + " failed {}" + ex);
 304             } finally {
 305                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 306             }
 307         } catch (Exception e) {
 308             logger.error("Unable to handle subnet {} removed to vpn {} {}", notification.getSubnetIp(), notification.getVpnName(), e);
 309         }
 310     }
 311
 312     @Override
 313     public void onSubnetUpdatedInVpn(SubnetUpdatedInVpn notification) {
 314         Uuid subnetId = notification.getSubnetId();
 315         String vpnName = notification.getVpnName();
 316         String subnetIp = notification.getSubnetIp();
 317         Long elanTag = notification.getElanTag();
 318
 319         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 320         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 321         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 322         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 323
 324         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 325                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 326         Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 327                 LogicalDatastoreType.OPERATIONAL,
 328                 subOpIdentifier);
 329         if (optionalSubs.isPresent()) {
 330             if (!notification.isExternalVpn()) {
 331                 SubnetDeletedFromVpnBuilder bldr = new SubnetDeletedFromVpnBuilder().setVpnName(vpnName);
 332                 bldr.setElanTag(elanTag).setExternalVpn(true).setSubnetIp(subnetIp).setSubnetId(subnetId);
 333                 onSubnetDeletedFromVpn(bldr.build());
 334             }
 335             // TODO(vivek): Something got updated, but we donot know what ?
 336         } else {
 337             if (notification.isExternalVpn()) {
 338                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
 339                 bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);;
 340                 onSubnetAddedToVpn(bldr.build());
 341             }
 342             // TODO(vivek): Something got updated, but we donot know what ?
 343         }
 344     }
 345
 346     @Override
 347     public void onPortAddedToSubnet(PortAddedToSubnet notification) {
 348         Uuid subnetId = notification.getSubnetId();
 349         Uuid portId = notification.getPortId();
 350
 351         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 352         //TODO(vivek): Change this to use more granularized lock at subnetId level
 353         try {
 354             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 355             try {
 356                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 357                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 358
 359                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 360                         subOpIdentifier);
 361                 if (!optionalSubs.isPresent()) {
 362                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 363                             " that is not in VPN, ignoring");
 364                     return;
 365                 }
 366                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,portId.getValue());
 367                 if (intfState == null) {
 368                     // Interface State not yet available
 369                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 370                     return;
 371                 }
 372                 BigInteger dpnId = BigInteger.ZERO;
 373                 try {
 374                     dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 375                 } catch (Exception e) {
 376                     logger.error("onSubnetAddedToVpn: Unable to obtain dpnId for interface {},",
 377                             " subnetroute inclusion for this interface failed with exception {}",
 378                             portId.getValue(), e);
 379                     return;
 380                 }
 381                 if (dpnId.equals(BigInteger.ZERO)) {
 382                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 383                     return;
 384                 }
 385                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 386                 if (intfState.getOperStatus() != OperStatus.Up) {
 387                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 388                     return;
 389                 }
 390                 logger.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 391                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 392                 if (subDpn == null) {
 393                     return;
 394                 }
 395                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 396                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 397                 subDpnList.add(subDpn);
 398                 subOpBuilder.setSubnetToDpn(subDpnList);
 399                 if (subOpBuilder.getNhDpnId()  == null) {
 400                     subOpBuilder.setNhDpnId(dpnId);
 401                 }
 402                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 403                 String rd = subOpBuilder.getVrfId();
 404                 String subnetIp = subOpBuilder.getSubnetCidr();
 405                 String vpnName = subOpBuilder.getVpnName();
 406                 Long elanTag = subOpBuilder.getElanTag();
 407                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) ||
 408                         (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 409                     try {
 410                         // Write the Subnet Route Entry to FIB
 411                         // Advertise BGP Route here and set route_adv_state to DONE
 412                         int label = getLabel(rd, subnetIp);
 413                         if (label == 0) {
 414                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling addition of port {} to subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
 415                             return;
 416                         }
 417                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
 418                         subOpBuilder.setRouteAdvState(TaskState.Done);
 419                     } catch (Exception ex) {
 420                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
 421                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 422                     }
 423                 }
 424                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 425                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 426                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 427
 428             } catch (Exception ex) {
 429                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 430                         subnetId.getValue() + " failed {}", ex);
 431             } finally {
 432                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 433             }
 434         } catch (Exception e) {
 435             logger.error("Unable to handle port {} added to subnet {} {}", portId.getValue(), subnetId.getValue(), e);
 436         }
 437     }
 438
 439     @Override
 440     public void onPortRemovedFromSubnet(PortRemovedFromSubnet notification) {
 441         Uuid subnetId = notification.getSubnetId();
 442         Uuid portId = notification.getPortId();
 443
 444         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 445         //TODO(vivek): Change this to use more granularized lock at subnetId level
 446         try {
 447             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 448             try {
 449                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 450                 if (portOpEntry == null) {
 451                     return;
 452                 }
 453                 BigInteger dpnId = portOpEntry.getDpnId();
 454                 if (dpnId == null) {
 455                     logger.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring", portId.getValue());
 456                     return;
 457                 }
 458                 logger.debug("onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 459                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 460                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 461                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 462                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 463                         subOpIdentifier);
 464                 if (!optionalSubs.isPresent()) {
 465                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 466                             " that is not in VPN, ignoring");
 467                     return;
 468                 }
 469                 SubnetOpDataEntry subOpEntry = null;
 470                 List<SubnetToDpn> subDpnList = null;
 471                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 472                 String rd = subOpBuilder.getVrfId();
 473                 String subnetIp = subOpBuilder.getSubnetCidr();
 474                 String vpnName = subOpBuilder.getVpnName();
 475                 Long elanTag = subOpBuilder.getElanTag();
 476                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 477                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 478                     // select another NhDpnId
 479                     if (last) {
 480                         logger.debug("onPortRemovedFromSubnet: Last port " + portId + " on the subnet: " +  subnetId.getValue());
 481                         // last port on this DPN, so we need to swap the NHDpnId
 482                         subDpnList = subOpBuilder.getSubnetToDpn();
 483                         if (subDpnList.isEmpty()) {
 484                             subOpBuilder.setNhDpnId(null);
 485                             try {
 486                                 // withdraw route from BGP
 487                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 488                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 489                             } catch (Exception ex) {
 490                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 491                                   subnetId.getValue() + " from BGP failed ", ex);
 492                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 493                             }
 494                         } else {
 495                             nhDpnId = subDpnList.get(0).getDpnId();
 496                             subOpBuilder.setNhDpnId(nhDpnId);
 497                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 498                             try {
 499                                 // Best effort Withdrawal of route from BGP for this subnet
 500                                 // Advertise the new NexthopIP to BGP for this subnet
 501                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 502                                 int label = getLabel(rd, subnetIp);
 503                                 if (label == 0) {
 504                                     logger.error("Unable to fetch label from Id Manager. Bailing out of handling removal  of port {} from subnet {} in vpn {}", portId.getValue(), subnetIp, vpnName);
 505                                     return;
 506                                 }
 507                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
 508                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 509                             } catch (Exception ex) {
 510                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
 511                                         " information for subnet " + subnetId.getValue() +
 512                                         " to BGP failed {}" + ex);
 513                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 514                             }
 515                         }
 516                     }
 517                 }
 518                 subOpEntry = subOpBuilder.build();
 519                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 520                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
 521             } catch (Exception ex) {
 522                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 523                         subnetId.getValue() + " failed {}" + ex);
 524             } finally {
 525                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 526             }
 527         } catch (Exception e) {
 528             logger.error("Unable to handle port {} removed from subnet {} {}", portId.getValue(), subnetId.getValue(), e);
 529         }
 530     }
 531
 532     public void onInterfaceUp(BigInteger dpnId, String intfName) {
 533         logger.info("onInterfaceUp: Port " + intfName);
 534         //TODO(vivek): Change this to use more granularized lock at subnetId level
 535         SubnetToDpn subDpn = null;
 536         PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 537         if (portOpEntry == null) {
 538             logger.info("onInterfaceUp: Port " + intfName  + "is part of a subnet not in VPN, ignoring");
 539             return;
 540         }
 541         if ((dpnId == null) || (dpnId == BigInteger.ZERO)) {
 542             dpnId = portOpEntry.getDpnId();
 543             if (dpnId == null) {
 544                 logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
 545             return;
 546             }
 547         }
 548         Uuid subnetId = portOpEntry.getSubnetId();
 549         try {
 550             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 551             try {
 552                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 553                     child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 554                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL,
 555                         subOpIdentifier);
 556                 if (!optionalSubs.isPresent()) {
 557                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 558                             " is not available");
 559                     return;
 560                 }
 561
 562                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 563                 logger.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 564                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 565                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 566                 if (subDpn == null) {
 567                     return;
 568                 }
 569                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 570                 subDpnList.add(subDpn);
 571                 subOpBuilder.setSubnetToDpn(subDpnList);
 572                 if (subOpBuilder.getNhDpnId()  == null) {
 573                     subOpBuilder.setNhDpnId(dpnId);
 574                 }
 575                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 576                 String rd = subOpBuilder.getVrfId();
 577                 String subnetIp = subOpBuilder.getSubnetCidr();
 578                 String vpnName = subOpBuilder.getVpnName();
 579                 Long elanTag = subOpBuilder.getElanTag();
 580                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) || (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 581                     try {
 582                         // Write the Subnet Route Entry to FIB
 583                         // Advertise BGP Route here and set route_adv_state to DONE
 584                         int label = getLabel(rd, subnetIp);
 585                         if (label == 0) {
 586                             logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface up event for port {} for subnet {} in vpn {}", intfName, subnetIp, vpnName);
 587                             return;
 588                         }
 589                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
 590                         subOpBuilder.setRouteAdvState(TaskState.Done);
 591                     } catch (Exception ex) {
 592                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
 593                                 subnetId.getValue() + " to BGP failed {}" + ex);
 594                     }
 595                 }
 596                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 597                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 598                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 599             } catch (Exception ex) {
 600                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 601                         subnetId.getValue() + " failed {}" + ex);
 602             } finally {
 603                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 604             }
 605         } catch (Exception e) {
 606             logger.error("Unable to handle interface up event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
 607         }
 608     }
 609
 610     public void onInterfaceDown(final BigInteger dpnId, final String interfaceName) {
 611         logger.info("onInterfaceDown: Port " + interfaceName);
 612         //TODO(vivek): Change this to use more granularized lock at subnetId level
 613         PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(interfaceName);
 614         if (portOpEntry == null) {
 615             logger.info("onInterfaceDown: Port " + interfaceName  + "is part of a subnet not in VPN, ignoring");
 616             return;
 617         }
 618         if ((dpnId  == null) ||(dpnId == BigInteger.ZERO)) {
 619             logger.error("onInterfaceDown: Unable to determine the DPNID for port " + interfaceName);
 620             return;
 621         }
 622         Uuid subnetId = portOpEntry.getSubnetId();
 623         try {
 624             VpnUtil.lockSubnet(lockManager, subnetId.getValue());
 625             try {
 626                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 627                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, interfaceName);
 628                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 629                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 630                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(dataBroker,
 631                         LogicalDatastoreType.OPERATIONAL,
 632                         subOpIdentifier);
 633                 if (!optionalSubs.isPresent()) {
 634                     logger.error("onInterfaceDown: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 635                             " is not available");
 636                     return;
 637                 }
 638                 SubnetOpDataEntry subOpEntry = null;
 639                 List<SubnetToDpn> subDpnList = null;
 640                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 641                 String rd = subOpBuilder.getVrfId();
 642                 String subnetIp = subOpBuilder.getSubnetCidr();
 643                 String vpnName = subOpBuilder.getVpnName();
 644                 Long elanTag = subOpBuilder.getElanTag();
 645                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 646                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 647                     // select another NhDpnId
 648                     if (last) {
 649                         logger.debug("onInterfaceDown: Last active port " + interfaceName + " on the subnet: " +  subnetId.getValue());
 650                         // last port on this DPN, so we need to swap the NHDpnId
 651                         subDpnList = subOpBuilder.getSubnetToDpn();
 652                         if (subDpnList.isEmpty()) {
 653                             subOpBuilder.setNhDpnId(null);
 654                             try {
 655                                 // Withdraw route from BGP for this subnet
 656                                 deleteSubnetRouteFromFib(rd, subnetIp, vpnName);
 657                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 658                             } catch (Exception ex) {
 659                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 660                                   subnetId.getValue() + " from BGP failed {}" + ex);
 661                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 662                             }
 663                         } else {
 664                             nhDpnId = subDpnList.get(0).getDpnId();
 665                             subOpBuilder.setNhDpnId(nhDpnId);
 666                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 667                             try {
 668                                 // Best effort Withdrawal of route from BGP for this subnet
 669                                 //withdrawSubnetRoutefromBgp(rd, subnetIp);
 670                                 int label = getLabel(rd, subnetIp);
 671                                 if (label == 0) {
 672                                     logger.error("Unable to fetch label from Id Manager. Bailing out of handling interface down event for port {} in subnet {} for vpn {}", interfaceName, subnetIp, vpnName);
 673                                     return;
 674                                 }
 675                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag, label, subnetId);
 676                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 677                             } catch (Exception ex) {
 678                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 679                                         subnetId.getValue() + " to BGP failed {}" + ex);
 680                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 681                             }
 682                         }
 683                     }
 684                 }
 685                 subOpEntry = subOpBuilder.build();
 686                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 687                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + interfaceName);
 688             } catch (Exception ex) {
 689                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 690                         subnetId.getValue() + " failed {}" + ex);
 691             } finally {
 692                 VpnUtil.unlockSubnet(lockManager, subnetId.getValue());
 693             }
 694         } catch (Exception e) {
 695             logger.error("Unable to handle interface down event for port {} in subnet {} {}", portOpEntry.getPortId(), subnetId.getValue(), e);
 696         }
 697     }
 698
 699     @Override
 700     public void onRouterAssociatedToVpn(RouterAssociatedToVpn notification) {
 701     }
 702
 703     @Override
 704     public void onRouterDisassociatedFromVpn(RouterDisassociatedFromVpn notification) {
 705     }
 706
 707     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 708                                      Long elanTag, int label, Uuid subnetId) throws Exception {
 709         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 710         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 711         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 712         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 713         String nexthopIp = null;
 714         try {
 715             nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, nhDpnId);
 716         } catch (Exception e) {
 717             logger.warn("Unable to find nexthopip for subnetroute subnetip {}", subnetIp);
 718             return;
 719         }
 720         if (nexthopIp != null) {
 721             VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, vpnName), subnetIp), VpnUtil.getPrefixToInterface(nhDpnId, subnetId.getValue(), subnetIp));
 722             vpnInterfaceManager.addSubnetRouteFibEntryToDS(rd, vpnName, subnetIp, nexthopIp, label, elanTag, nhDpnId, null);
 723             try {
 724                 // BGPManager (inside ODL) requires a withdraw followed by advertise
 725                 // due to bugs with ClusterDataChangeListener used by BGPManager.
 726                 //bgpManager.withdrawPrefix(rd, subnetIp);
 727                 bgpManager.advertisePrefix(rd, subnetIp, Arrays.asList(nexthopIp), label);
 728             } catch (Exception e) {
 729                 logger.error("Fail: Subnet route not advertised for rd {} subnetIp {}", rd, subnetIp, e);
 730                 throw e;
 731             }
 732         } else {
 733             logger.warn("The nexthopip is empty for subnetroute subnetip {}, ignoring fib route addition", subnetIp);
 734         }
 735     }
 736
 737     private int getLabel(String rd, String subnetIp) {
 738         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 739                                         VpnUtil.getNextHopLabelKey(rd, subnetIp));
 740         logger.trace("Allocated subnetroute label {} for rd {} prefix {}", label, rd, subnetIp);
 741         return label;
 742     }
 743
 744     private void deleteSubnetRouteFromFib(String rd, String subnetIp, String vpnName) throws Exception {
 745         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 746         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 747         vpnInterfaceManager.deleteSubnetRouteFibEntryFromDS(rd, subnetIp, vpnName);
 748         try {
 749             bgpManager.withdrawPrefix(rd, subnetIp);
 750         } catch (Exception e) {
 751             logger.error("Fail: Subnet route not withdrawn for rd {} subnetIp {}", rd, subnetIp, e);
 752             throw e;
 753         }
 754     }
 755 }
 756