vpnmanager/vpnmanager-impl/src/main/java/org/opendaylight/vpnservice/VpnSubnetRouteHandler.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.vpnservice;
   9
  10 import java.util.ArrayList;
  11 import java.util.List;
  12 import java.util.Map;
  13 import java.util.HashMap;
  14
  15 import com.google.common.base.Preconditions;
  16 import org.opendaylight.bgpmanager.api.IBgpManager;
  17 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  18 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  19 import org.opendaylight.vpnservice.mdsalutil.MDSALUtil;
  20 import org.opendaylight.vpnservice.utilities.InterfaceUtils;
  21 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  22 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  23 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.*;
  24 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.port.op.data.PortOpDataEntry;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.port.op.data.PortOpDataEntryKey;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.rd.to.elan.op.RdToElanOpEntry;
  27 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.rd.to.elan.op.RdToElanOpEntryBuilder;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.rd.to.elan.op.RdToElanOpEntryKey;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntry;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryKey;
  31 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.subnet.op.data.SubnetOpDataEntryBuilder;
  32 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.idmanager.rev150403.IdManagerService;
  33 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.*;
  34 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.subnetmaps.Subnetmap;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.vpnservice.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.SubnetToDpn;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.l3vpn.rev130911.subnet.op.data.subnet.op.data.entry.subnet.to.dpn.VpnInterfaces;
  38 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  39 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  40
  41 import java.math.BigInteger;
  42
  43 import org.slf4j.Logger;
  44 import org.slf4j.LoggerFactory;
  45
  46 import com.google.common.base.Optional;
  47
  48
  49 public class VpnSubnetRouteHandler implements NeutronvpnListener {
  50     private static final Logger logger = LoggerFactory.getLogger(VpnSubnetRouteHandler.class);
  51
  52     private final DataBroker broker;
  53     private SubnetOpDpnManager subOpDpnManager;
  54     private final IBgpManager bgpManager;
  55     private IdManagerService idManager;
  56     private VpnInterfaceManager vpnInterfaceManager;
  57
  58     public VpnSubnetRouteHandler(final DataBroker db, IBgpManager bgpManager, VpnInterfaceManager vpnIntfManager) {
  59         broker = db;
  60         subOpDpnManager = new SubnetOpDpnManager(broker);
  61         this.bgpManager = bgpManager;
  62         this.vpnInterfaceManager = vpnIntfManager;
  63     }
  64
  65     public void setIdManager(IdManagerService idManager) {
  66         this.idManager = idManager;
  67     }
  68
  69     @Override
  70     public void onSubnetAddedToVpn(SubnetAddedToVpn notification) {
  71         if (!notification.isExternalVpn()) {
  72             return;
  73         }
  74
  75         Uuid subnetId = notification.getSubnetId();
  76         String vpnName = notification.getVpnName();
  77         String subnetIp = notification.getSubnetIp();
  78         Long elanTag = notification.getElanTag();
  79
  80         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
  81         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
  82         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
  83         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
  84
  85         logger.info("onSubnetAddedToVpn: Subnet" + subnetId.getValue() + " being added to vpn");
  86         //TODO(vivek): Change this to use more granularized lock at subnetId level
  87         synchronized (this) {
  88             try {
  89                 //Create and add SubnetOpDataEntry object for this subnet to the SubnetOpData container
  90                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
  91                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
  92                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
  93                         LogicalDatastoreType.OPERATIONAL,
  94                         subOpIdentifier);
  95                 if (optionalSubs.isPresent()) {
  96                     logger.error("onSubnetAddedToVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
  97                             " already detected to be present");
  98                     return;
  99                 }
 100                 logger.debug("onSubnetAddedToVpn: Creating new SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 101                 Map<BigInteger, SubnetToDpn> subDpnMap = new HashMap<BigInteger, SubnetToDpn>();
 102                 SubnetOpDataEntry subOpEntry = null;
 103                 BigInteger dpnId = null;
 104                 BigInteger nhDpnId = null;
 105                 SubnetToDpn subDpn = null;
 106
 107                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder().setKey(new SubnetOpDataEntryKey(subnetId));
 108                 subOpBuilder.setSubnetId(subnetId);
 109                 subOpBuilder.setSubnetCidr(subnetIp);
 110                 String rd = VpnUtil.getVpnRdFromVpnInstanceConfig(broker, vpnName);
 111                 if (rd == null) {
 112                     logger.error("onSubnetAddedToVpn: The VPN Instance name " + notification.getVpnName() + " does not have RD ");
 113                     return;
 114                 }
 115                 subOpBuilder.setVrfId(rd);
 116                 subOpBuilder.setVpnName(vpnName);
 117                 subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>());
 118                 subOpBuilder.setRouteAdvState(TaskState.Na);
 119                 subOpBuilder.setElanTag(elanTag);
 120
 121                 // First recover set of ports available in this subnet
 122                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 123                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 124                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
 125                 if (!sm.isPresent()) {
 126                     logger.error("onSubnetAddedToVpn: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 127                     return;
 128                 }
 129                 Subnetmap subMap = sm.get();
 130                 List<Uuid> portList = subMap.getPortList();
 131                 if (portList != null) {
 132                     for (Uuid port: portList) {
 133                         Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,port.getValue());
 134                         if (intfState != null) {
 135                             dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 136                             if (dpnId == null) {
 137                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not assigned DPN yet, ignoring ");
 138                                 continue;
 139                             }
 140                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, dpnId);
 141                             if (intfState.getOperStatus() != OperStatus.Up) {
 142                                 logger.info("onSubnetAddedToVpn: Port " + port.getValue() + " is not UP yet, ignoring ");
 143                                 continue;
 144                             }
 145                             subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, port.getValue());
 146                             if (intfState.getOperStatus() == OperStatus.Up) {
 147                                 // port is UP
 148                                 subDpnMap.put(dpnId, subDpn);
 149                                 if (nhDpnId == null) {
 150                                     nhDpnId = dpnId;
 151                                 }
 152                             }
 153                         } else {
 154                             subOpDpnManager.addPortOpDataEntry(port.getValue(), subnetId, null);
 155                         }
 156                     }
 157                     if (subDpnMap.size() > 0) {
 158                         subOpBuilder.setSubnetToDpn(new ArrayList<SubnetToDpn>(subDpnMap.values()));
 159                     }
 160                 }
 161
 162                 if (nhDpnId != null) {
 163                     subOpBuilder.setNhDpnId(nhDpnId);
 164                     try {
 165                         /*
 166                         Write the subnet route entry to the FIB.
 167                         And also advertise the subnet route entry via BGP.
 168                         */
 169                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag);
 170                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag);
 171                         subOpBuilder.setRouteAdvState(TaskState.Done);
 172                     } catch (Exception ex) {
 173                         logger.error("onSubnetAddedToVpn: FIB rules and Advertising nhDpnId " + nhDpnId +
 174                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 175                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 176                     }
 177                 } else {
 178                     try {
 179                         /*
 180                         Write the subnet route entry to the FIB.
 181                         NOTE: Will not advertise to BGP as NextHopDPN is not available yet.
 182                         */
 183                         addSubnetRouteToFib(rd, subnetIp, null, vpnName, elanTag);
 184                     } catch (Exception ex) {
 185                         logger.error("onSubnetAddedToVpn: FIB rules writing for subnet {} with exception {} " +
 186                                 subnetId.getValue(), ex);
 187                         subOpBuilder.setRouteAdvState(TaskState.Pending);
 188                     }
 189                 }
 190
 191                 subOpEntry = subOpBuilder.build();
 192                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 193                 logger.info("onSubnetAddedToVpn: Added subnetopdataentry to OP Datastore for subnet " + subnetId.getValue());
 194             } catch (Exception ex) {
 195                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 196                         subnetId.getValue() + " failed {}", ex);
 197             } finally {
 198             }
 199         }
 200     }
 201
 202     @Override
 203     public void onSubnetDeletedFromVpn(SubnetDeletedFromVpn notification) {
 204         Uuid subnetId = notification.getSubnetId();
 205
 206         if (!notification.isExternalVpn()) {
 207             return;
 208         }
 209         logger.info("onSubnetDeletedFromVpn: Subnet" + subnetId.getValue() + " being removed to vpn");
 210         //TODO(vivek): Change this to use more granularized lock at subnetId level
 211         synchronized (this) {
 212             try {
 213                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 214                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 215                 logger.trace(" Removing the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 216                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 217                         LogicalDatastoreType.OPERATIONAL,
 218                         subOpIdentifier);
 219                 if (!optionalSubs.isPresent()) {
 220                     logger.error("onSubnetDeletedFromVpn: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 221                             " not available in datastore");
 222                     return;
 223                 }
 224                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 225                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 226                 for (SubnetToDpn subDpn: subDpnList) {
 227                     List<VpnInterfaces> vpnIntfList = subDpn.getVpnInterfaces();
 228                     for (VpnInterfaces vpnIntf: vpnIntfList) {
 229                         subOpDpnManager.removePortOpDataEntry(vpnIntf.getInterfaceName());
 230                     }
 231                 }
 232                 //Removing Stale Ports in portOpData
 233                 InstanceIdentifier<Subnetmap> subMapid = InstanceIdentifier.builder(Subnetmaps.class).
 234                         child(Subnetmap.class, new SubnetmapKey(subnetId)).build();
 235                 Optional<Subnetmap> sm = VpnUtil.read(broker, LogicalDatastoreType.CONFIGURATION, subMapid);
 236                 if (!sm.isPresent()) {
 237                     logger.error("Stale ports removal: Unable to retrieve subnetmap entry for subnet : " + subnetId);
 238                 }
 239                 Subnetmap subMap = sm.get();
 240                 List<Uuid> portList = subMap.getPortList();
 241                 if(portList!=null){
 242                     InstanceIdentifier<PortOpData> portOpIdentifier = InstanceIdentifier.builder(PortOpData.class).build();
 243                     Optional<PortOpData> optionalPortOp = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier);
 244                     if(!optionalPortOp.isPresent()){
 245                         logger.error("Stale ports removal: Cannot delete port. Not available in data store");
 246                         return;
 247                     } else{
 248                         PortOpData portOpData = optionalPortOp.get();
 249                         List<PortOpDataEntry> portOpDataList = portOpData.getPortOpDataEntry();
 250                         if(portOpDataList!=null){
 251                             for(PortOpDataEntry portOpDataListEntry :  portOpDataList){
 252                                 if(portList.contains(new Uuid(portOpDataListEntry.getPortId()))){
 253                                     logger.trace("Removing stale port: " + portOpDataListEntry + "for dissociated subnetId: " + subnetId);
 254                                     MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, portOpIdentifier.
 255                                             child(PortOpDataEntry.class, new PortOpDataEntryKey(portOpDataListEntry.getKey())));
 256                                 }
 257                             }
 258                         }
 259                     }
 260                 }
 261
 262                 String rd = subOpBuilder.getVrfId();
 263                 String subnetIp = subOpBuilder.getSubnetCidr();
 264                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 265                 MDSALUtil.syncDelete(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier);
 266                 logger.trace("Removed subnetopdataentry successfully to CONFIG Datastore");
 267                 try {
 268                     //Withdraw the routes for all the interfaces on this subnet
 269                     //Remove subnet route entry from FIB
 270                     withdrawSubnetRoutefromBgp(rd, subnetIp);
 271                     deleteSubnetRouteFromFib(rd, subnetIp);
 272                 } catch (Exception ex) {
 273                     logger.error("onSubnetAddedToVpn: Withdrawing routes from BGP for subnet " +
 274                             subnetId.getValue() + " failed {}" + ex);
 275                 }
 276             } catch (Exception ex) {
 277                 logger.error("Removal of SubnetOpDataEntry for subnet " +
 278                         subnetId.getValue() + " failed {}" + ex);
 279             } finally {
 280             }
 281         }
 282     }
 283
 284     @Override
 285     public void onSubnetUpdatedInVpn(SubnetUpdatedInVpn notification) {
 286         Uuid subnetId = notification.getSubnetId();
 287         String vpnName = notification.getVpnName();
 288         String subnetIp = notification.getSubnetIp();
 289         Long elanTag = notification.getElanTag();
 290
 291         Preconditions.checkNotNull(subnetId, "SubnetId cannot be null or empty!");
 292         Preconditions.checkNotNull(subnetIp, "SubnetPrefix cannot be null or empty!");
 293         Preconditions.checkNotNull(vpnName, "VpnName cannot be null or empty!");
 294         Preconditions.checkNotNull(elanTag, "ElanTag cannot be null or empty!");
 295
 296         InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 297                 child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 298         Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 299                 LogicalDatastoreType.OPERATIONAL,
 300                 subOpIdentifier);
 301         if (optionalSubs.isPresent()) {
 302             if (!notification.isExternalVpn()) {
 303                 SubnetDeletedFromVpnBuilder bldr = new SubnetDeletedFromVpnBuilder().setVpnName(vpnName);
 304                 bldr.setElanTag(elanTag).setExternalVpn(true).setSubnetIp(subnetIp).setSubnetId(subnetId);
 305                 onSubnetDeletedFromVpn(bldr.build());
 306             }
 307             // TODO(vivek): Something got updated, but we donot know what ?
 308         } else {
 309             if (notification.isExternalVpn()) {
 310                 SubnetAddedToVpnBuilder bldr = new SubnetAddedToVpnBuilder().setVpnName(vpnName).setElanTag(elanTag);
 311                 bldr.setSubnetIp(subnetIp).setSubnetId(subnetId).setExternalVpn(true);;
 312                 onSubnetAddedToVpn(bldr.build());
 313             }
 314             // TODO(vivek): Something got updated, but we donot know what ?
 315         }
 316     }
 317
 318     @Override
 319     public void onPortAddedToSubnet(PortAddedToSubnet notification) {
 320         Uuid subnetId = notification.getSubnetId();
 321         Uuid portId = notification.getPortId();
 322
 323         logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " being added to subnet " + subnetId.getValue());
 324         //TODO(vivek): Change this to use more granularized lock at subnetId level
 325         synchronized (this) {
 326             try {
 327                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 328                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 329
 330                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 331                         subOpIdentifier);
 332                 if (!optionalSubs.isPresent()) {
 333                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 334                             " that is not in VPN, ignoring");
 335                     return;
 336                 }
 337                 Interface intfState = InterfaceUtils.getInterfaceStateFromOperDS(broker,portId.getValue());
 338                 if (intfState == null) {
 339                     // Interface State not yet available
 340                     subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, null);
 341                     return;
 342                 }
 343                 BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 344                 if (dpnId == null) {
 345                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not assigned DPN yet, ignoring ");
 346                     return;
 347                 }
 348                 subOpDpnManager.addPortOpDataEntry(portId.getValue(), subnetId, dpnId);
 349                 if (intfState.getOperStatus() != OperStatus.Up) {
 350                     logger.info("onPortAddedToSubnet: Port " + portId.getValue() + " is not UP yet, ignoring ");
 351                     return;
 352                 }
 353                 logger.debug("onPortAddedToSubnet: Updating the SubnetOpDataEntry node for subnet: " + subnetId.getValue());
 354                 SubnetToDpn subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, portId.getValue());
 355                 if (subDpn == null) {
 356                     return;
 357                 }
 358                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 359                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 360                 subDpnList.add(subDpn);
 361                 subOpBuilder.setSubnetToDpn(subDpnList);
 362                 if (subOpBuilder.getNhDpnId()  == null) {
 363                     subOpBuilder.setNhDpnId(dpnId);
 364                 }
 365                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 366                 String rd = subOpBuilder.getVrfId();
 367                 String subnetIp = subOpBuilder.getSubnetCidr();
 368                 String vpnName = subOpBuilder.getVpnName();
 369                 Long elanTag = subOpBuilder.getElanTag();
 370                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) ||
 371                         (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 372                     try {
 373                         // Write the Subnet Route Entry to FIB
 374                         // Advertise BGP Route here and set route_adv_state to DONE
 375                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag);
 376                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag);
 377                         subOpBuilder.setRouteAdvState(TaskState.Done);
 378                     } catch (Exception ex) {
 379                         logger.error("onPortAddedToSubnet: Advertising NextHopDPN "+ nhDpnId +
 380                                 " information for subnet " + subnetId.getValue() + " to BGP failed {}", ex);
 381                     }
 382                 }
 383                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 384                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 385                 logger.info("onPortAddedToSubnet: Updated subnetopdataentry to OP Datastore for port " + portId.getValue());
 386
 387             } catch (Exception ex) {
 388                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 389                         subnetId.getValue() + " failed {}", ex);
 390             } finally {
 391             }
 392         }
 393     }
 394
 395     @Override
 396     public void onPortRemovedFromSubnet(PortRemovedFromSubnet notification) {
 397         Uuid subnetId = notification.getSubnetId();
 398         Uuid portId = notification.getPortId();
 399
 400         logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " being removed from subnet " + subnetId.getValue());
 401         //TODO(vivek): Change this to use more granularized lock at subnetId level
 402         synchronized (this) {
 403             try {
 404                 PortOpDataEntry portOpEntry = subOpDpnManager.removePortOpDataEntry(portId.getValue());
 405                 if (portOpEntry == null) {
 406                     return;
 407                 }
 408                 BigInteger dpnId = portOpEntry.getDpnId();
 409                 if (dpnId == null) {
 410                     logger.debug("onPortRemovedFromSubnet:  Port {} does not have a DPNId associated, ignoring", portId.getValue());
 411                     return;
 412                 }
 413                 logger.debug("onPortRemovedFromSubnet: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 414                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, portId.getValue());
 415                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 416                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 417                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 418                         subOpIdentifier);
 419                 if (!optionalSubs.isPresent()) {
 420                     logger.info("onPortRemovedFromSubnet: Port " + portId.getValue() + " is part of a subnet " + subnetId.getValue() +
 421                             " that is not in VPN, ignoring");
 422                     return;
 423                 }
 424                 SubnetOpDataEntry subOpEntry = null;
 425                 List<SubnetToDpn> subDpnList = null;
 426                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 427                 String rd = subOpBuilder.getVrfId();
 428                 String subnetIp = subOpBuilder.getSubnetCidr();
 429                 String vpnName = subOpBuilder.getVpnName();
 430                 Long elanTag = subOpBuilder.getElanTag();
 431                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 432                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 433                     // select another NhDpnId
 434                     if (last) {
 435                         logger.debug("onPortRemovedFromSubnet: Last port " + portId + " on the subnet: " +  subnetId.getValue());
 436                         // last port on this DPN, so we need to swap the NHDpnId
 437                         subDpnList = subOpBuilder.getSubnetToDpn();
 438                         if (subDpnList.isEmpty()) {
 439                             subOpBuilder.setNhDpnId(null);
 440                             try {
 441                                 // withdraw route from BGP
 442                                 deleteSubnetRouteFromFib(rd, subnetIp);
 443                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 444                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 445                             } catch (Exception ex) {
 446                                 logger.error("onPortRemovedFromSubnet: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 447                                         subnetId.getValue() + " from BGP failed ", ex);
 448                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 449                             }
 450                         } else {
 451                             nhDpnId = subDpnList.get(0).getDpnId();
 452                             subOpBuilder.setNhDpnId(nhDpnId);
 453                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 454                             try {
 455                                 // Best effort Withdrawal of route from BGP for this subnet
 456                                 // Advertise the new NexthopIP to BGP for this subnet
 457                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 458                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag);
 459                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag);
 460                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 461                             } catch (Exception ex) {
 462                                 logger.error("onPortRemovedFromSubnet: Swapping Withdrawing NextHopDPN " + dpnId +
 463                                         " information for subnet " + subnetId.getValue() +
 464                                         " to BGP failed {}" + ex);
 465                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 466                             }
 467                         }
 468                     }
 469                 }
 470                 subOpEntry = subOpBuilder.build();
 471                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 472                 logger.info("onPortRemovedFromSubnet: Updated subnetopdataentry to OP Datastore removing port " + portId.getValue());
 473             } catch (Exception ex) {
 474                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 475                         subnetId.getValue() + " failed {}" + ex);
 476             } finally {
 477             }
 478         }
 479     }
 480
 481     public void onInterfaceUp(Interface intfState) {
 482
 483         logger.info("onInterfaceUp: Port " + intfState.getName());
 484         //TODO(vivek): Change this to use more granularized lock at subnetId level
 485         synchronized (this) {
 486             SubnetToDpn subDpn = null;
 487             String intfName = intfState.getName();
 488             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 489             if (portOpEntry == null) {
 490                 logger.info("onInterfaceUp: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 491                 return;
 492             }
 493             BigInteger dpnId = portOpEntry.getDpnId();
 494             if (dpnId  == null) {
 495                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 496                 if (dpnId == null) {
 497                     logger.error("onInterfaceUp: Unable to determine the DPNID for port " + intfName);
 498                     return;
 499                 }
 500             }
 501             Uuid subnetId = portOpEntry.getSubnetId();
 502             try {
 503                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 504                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 505                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL,
 506                         subOpIdentifier);
 507                 if (!optionalSubs.isPresent()) {
 508                     logger.error("onInterfaceUp: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 509                             " is not available");
 510                     return;
 511                 }
 512
 513                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 514                 logger.debug("onInterfaceUp: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 515                 subOpDpnManager.addPortOpDataEntry(intfName, subnetId, dpnId);
 516                 subDpn = subOpDpnManager.addInterfaceToDpn(subnetId, dpnId, intfName);
 517                 if (subDpn == null) {
 518                     return;
 519                 }
 520                 List<SubnetToDpn> subDpnList = subOpBuilder.getSubnetToDpn();
 521                 subDpnList.add(subDpn);
 522                 subOpBuilder.setSubnetToDpn(subDpnList);
 523                 if (subOpBuilder.getNhDpnId()  == null) {
 524                     subOpBuilder.setNhDpnId(dpnId);
 525                 }
 526                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 527                 String rd = subOpBuilder.getVrfId();
 528                 String subnetIp = subOpBuilder.getSubnetCidr();
 529                 String vpnName = subOpBuilder.getVpnName();
 530                 Long elanTag = subOpBuilder.getElanTag();
 531                 if ((subOpBuilder.getRouteAdvState() == TaskState.Pending) || (subOpBuilder.getRouteAdvState() == TaskState.Na)) {
 532                     try {
 533                         // Write the Subnet Route Entry to FIB
 534                         // Advertise BGP Route here and set route_adv_state to DONE
 535                         addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag);
 536                         advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag);
 537                         subOpBuilder.setRouteAdvState(TaskState.Done);
 538                     } catch (Exception ex) {
 539                         logger.error("onInterfaceUp: Advertising NextHopDPN " + nhDpnId + " information for subnet " +
 540                                 subnetId.getValue() + " to BGP failed {}" + ex);
 541                     }
 542                 }
 543                 SubnetOpDataEntry subOpEntry = subOpBuilder.build();
 544                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 545                 logger.info("onInterfaceUp: Updated subnetopdataentry to OP Datastore port up " + intfName);
 546             } catch (Exception ex) {
 547                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 548                         subnetId.getValue() + " failed {}" + ex);
 549             } finally {
 550             }
 551         }
 552     }
 553
 554     public void onInterfaceDown(Interface intfState) {
 555         logger.info("onInterfaceDown: Port " + intfState.getName());
 556         //TODO(vivek): Change this to use more granularized lock at subnetId level
 557         synchronized (this) {
 558             String intfName = intfState.getName();
 559             PortOpDataEntry portOpEntry = subOpDpnManager.getPortOpDataEntry(intfName);
 560             if (portOpEntry == null) {
 561                 logger.info("onInterfaceDown: Port " + intfState.getName()  + "is part of a subnet not in VPN, ignoring");
 562                 return;
 563             }
 564             BigInteger dpnId = portOpEntry.getDpnId();
 565             if (dpnId  == null) {
 566                 dpnId = InterfaceUtils.getDpIdFromInterface(intfState);
 567                 if (dpnId == null) {
 568                     logger.error("onInterfaceDown: Unable to determine the DPNID for port " + intfName);
 569                     return;
 570                 }
 571             }
 572             Uuid subnetId = portOpEntry.getSubnetId();
 573             try {
 574                 logger.debug("onInterfaceDown: Updating the SubnetOpDataEntry node for subnet: " +  subnetId.getValue());
 575                 boolean last = subOpDpnManager.removeInterfaceFromDpn(subnetId, dpnId, intfName);
 576                 InstanceIdentifier<SubnetOpDataEntry> subOpIdentifier = InstanceIdentifier.builder(SubnetOpData.class).
 577                         child(SubnetOpDataEntry.class, new SubnetOpDataEntryKey(subnetId)).build();
 578                 Optional<SubnetOpDataEntry> optionalSubs = VpnUtil.read(broker,
 579                         LogicalDatastoreType.OPERATIONAL,
 580                         subOpIdentifier);
 581                 if (!optionalSubs.isPresent()) {
 582                     logger.error("onInterfaceDown: SubnetOpDataEntry for subnet " + subnetId.getValue() +
 583                             " is not available");
 584                     return;
 585                 }
 586                 SubnetOpDataEntry subOpEntry = null;
 587                 List<SubnetToDpn> subDpnList = null;
 588                 SubnetOpDataEntryBuilder subOpBuilder = new SubnetOpDataEntryBuilder(optionalSubs.get());
 589                 String rd = subOpBuilder.getVrfId();
 590                 String subnetIp = subOpBuilder.getSubnetCidr();
 591                 String vpnName = subOpBuilder.getVpnName();
 592                 Long elanTag = subOpBuilder.getElanTag();
 593                 BigInteger nhDpnId = subOpBuilder.getNhDpnId();
 594                 if ((nhDpnId != null) && (nhDpnId.equals(dpnId))) {
 595                     // select another NhDpnId
 596                     if (last) {
 597                         logger.debug("onInterfaceDown: Last active port " + intfState.getName() + " on the subnet: " +  subnetId.getValue());
 598                         // last port on this DPN, so we need to swap the NHDpnId
 599                         subDpnList = subOpBuilder.getSubnetToDpn();
 600                         if (subDpnList.isEmpty()) {
 601                             subOpBuilder.setNhDpnId(null);
 602                             try {
 603                                 // Withdraw route from BGP for this subnet
 604                                 deleteSubnetRouteFromFib(rd, subnetIp);
 605                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 606                                 subOpBuilder.setRouteAdvState(TaskState.Na);
 607                             } catch (Exception ex) {
 608                                 logger.error("onInterfaceDown: Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 609                                         subnetId.getValue() + " from BGP failed {}" + ex);
 610                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 611                             }
 612                         } else {
 613                             nhDpnId = subDpnList.get(0).getDpnId();
 614                             subOpBuilder.setNhDpnId(nhDpnId);
 615                             logger.debug("onInterfaceDown: Swapping the Designated DPN to " + nhDpnId + " for subnet " + subnetId.getValue());
 616                             try {
 617                                 // Best effort Withdrawal of route from BGP for this subnet
 618                                 withdrawSubnetRoutefromBgp(rd, subnetIp);
 619                                 addSubnetRouteToFib(rd, subnetIp, nhDpnId, vpnName, elanTag);
 620                                 advertiseSubnetRouteToBgp(rd, subnetIp, nhDpnId, vpnName, elanTag);
 621                                 subOpBuilder.setRouteAdvState(TaskState.Done);
 622                             } catch (Exception ex) {
 623                                 logger.error("onInterfaceDown: Swapping Withdrawing NextHopDPN " + dpnId + " information for subnet " +
 624                                         subnetId.getValue() + " to BGP failed {}" + ex);
 625                                 subOpBuilder.setRouteAdvState(TaskState.Pending);
 626                             }
 627                         }
 628                     }
 629                 }
 630                 subOpEntry = subOpBuilder.build();
 631                 MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, subOpIdentifier, subOpEntry);
 632                 logger.info("onInterfaceDown: Updated subnetopdataentry to OP Datastore port down " + intfName);
 633             } catch (Exception ex) {
 634                 logger.error("Creation of SubnetOpDataEntry for subnet " +
 635                         subnetId.getValue() + " failed {}" + ex);
 636             } finally {
 637             }
 638         }
 639     }
 640
 641     private static void setRdToElanOpEntry(DataBroker broker,
 642                                            String rd, String subnetIp, String nextHopIp, String vpnName,
 643                                            Long elanTag) {
 644         RdToElanOpEntryBuilder rdElanBuilder = null;
 645         RdToElanOpEntry rdElan = null;
 646
 647         try {
 648             InstanceIdentifier<RdToElanOpEntry> rdIdentifier = InstanceIdentifier.builder(RdToElanOp.class).
 649                     child(RdToElanOpEntry.class, new RdToElanOpEntryKey(rd, subnetIp)).build();
 650             Optional<RdToElanOpEntry> optionalRd = VpnUtil.read(broker, LogicalDatastoreType.OPERATIONAL, rdIdentifier);
 651             if (!optionalRd.isPresent()) {
 652                 // Create PortOpDataEntry only if not present
 653                 rdElanBuilder = new RdToElanOpEntryBuilder().setKey(new RdToElanOpEntryKey(rd,subnetIp));
 654                 rdElanBuilder.setRd(rd).setSubnetIp(subnetIp).setNextHopIp(nextHopIp);
 655                 rdElanBuilder.setElanTag(elanTag);
 656                 rdElanBuilder.setVpnName(vpnName);
 657                 rdElan = rdElanBuilder.build();
 658             } else {
 659                 rdElanBuilder = new RdToElanOpEntryBuilder(optionalRd.get());
 660                 rdElanBuilder.setRd(rd).setSubnetIp(subnetIp).setNextHopIp(nextHopIp);
 661                 rdElanBuilder.setElanTag(elanTag);
 662                 rdElanBuilder.setVpnName(vpnName);
 663                 rdElan = rdElanBuilder.build();
 664             }
 665             MDSALUtil.syncWrite(broker, LogicalDatastoreType.OPERATIONAL, rdIdentifier, rdElan);
 666             logger.info("Creating RdToElan entry Rd {} SubnetIp {} NextHopIp {} Elan {} " ,rd, subnetIp, nextHopIp, elanTag);
 667         } catch (Exception ex) {
 668             logger.error("Exception when creating RdToElan entry {}" + ex);
 669         } finally {
 670         }
 671     }
 672
 673     private void addSubnetRouteToFib(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 674                                      Long elanTag) {
 675         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 676         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 677         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 678         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 679         String nexthopIp = null;
 680         if (nhDpnId != null) {
 681             nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 682         }
 683         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 684                 VpnUtil.getNextHopLabelKey(rd, subnetIp));
 685         setRdToElanOpEntry(broker, rd, subnetIp, nexthopIp, vpnName, elanTag);
 686         vpnInterfaceManager.addFibEntryToDS(rd, subnetIp, nexthopIp, label);
 687     }
 688
 689     private void deleteSubnetRouteFromFib(String rd, String subnetIp) {
 690         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 691         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 692         vpnInterfaceManager.removeFibEntryFromDS(rd, subnetIp);
 693     }
 694
 695     private void advertiseSubnetRouteToBgp(String rd, String subnetIp, BigInteger nhDpnId, String vpnName,
 696                                            Long elanTag) throws Exception {
 697         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 698         Preconditions.checkNotNull(subnetIp, "SubnetRouteIp cannot be null or empty!");
 699         Preconditions.checkNotNull(elanTag, "elanTag cannot be null or empty!");
 700         Preconditions.checkNotNull(nhDpnId, "nhDpnId cannot be null or empty!");
 701         Preconditions.checkNotNull(vpnName, "vpnName cannot be null or empty!");
 702         String nexthopIp = null;
 703         nexthopIp = InterfaceUtils.getEndpointIpAddressForDPN(broker, nhDpnId);
 704         if (nexthopIp == null) {
 705             logger.error("createSubnetRouteInVpn: Unable to obtain endpointIp address for DPNId " + nhDpnId);
 706             throw new Exception("Unable to obtain endpointIp address for DPNId " + nhDpnId);
 707         }
 708         int label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
 709                 VpnUtil.getNextHopLabelKey(rd, subnetIp));
 710         setRdToElanOpEntry(broker, rd, subnetIp, nexthopIp, vpnName, elanTag);
 711         try {
 712             bgpManager.advertisePrefix(rd, subnetIp, nexthopIp, label);
 713         } catch (Exception e) {
 714             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 715             throw e;
 716         }
 717     }
 718
 719     private void withdrawSubnetRoutefromBgp(String rd, String subnetIp) throws Exception {
 720         Preconditions.checkNotNull(rd, "RouteDistinguisher cannot be null or empty!");
 721         Preconditions.checkNotNull(subnetIp, "SubnetIp cannot be null or empty!");
 722         try {
 723             bgpManager.withdrawPrefix(rd, subnetIp);
 724         } catch (Exception e) {
 725             logger.error("Subnet route not advertised for rd " + rd + " failed ", e);
 726             throw e;
 727         }
 728     }
 729 }
 730