openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.controller.sal.core.Node;
  24 import org.opendaylight.controller.sal.utils.HexEncode;
  25 import org.opendaylight.controller.sal.utils.Status;
  26 import org.opendaylight.controller.sal.utils.StatusCode;
  27 import org.opendaylight.ovsdb.lib.notation.Row;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.MultiTenantAwareRouter;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  33 import org.opendaylight.ovsdb.compatibility.plugin.api.OvsdbConfigurationService;
  34 import org.opendaylight.ovsdb.compatibility.plugin.api.OvsdbConnectionService;
  35 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  41 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  42 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  43 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  44
  45 import com.google.common.base.Preconditions;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48
  49 import java.net.InetAddress;
  50 import java.net.UnknownHostException;
  51 import java.util.HashMap;
  52 import java.util.HashSet;
  53 import java.util.List;
  54 import java.util.Map;
  55 import java.util.Set;
  56
  57 /**
  58  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  59  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  60  * as well as the multi-tenant router forwarding provider.
  61  */
  62 public class NeutronL3Adapter {
  63
  64     /**
  65      * Logger instance.
  66      */
  67     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  68
  69     // The implementation for each of these services is resolved by the OSGi Service Manager
  70     private volatile ConfigurationService configurationService;
  71     private volatile TenantNetworkManager tenantNetworkManager;
  72     private volatile NetworkingProviderManager networkingProviderManager;
  73     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  74     private volatile OvsdbConnectionService connectionService;
  75     private volatile INeutronNetworkCRUD neutronNetworkCache;
  76     private volatile INeutronSubnetCRUD neutronSubnetCache;
  77     private volatile INeutronPortCRUD neutronPortCache;
  78     private volatile MultiTenantAwareRouter multiTenantAwareRouter;
  79     private volatile L3ForwardingProvider l3ForwardingProvider;
  80     private volatile InboundNatProvider inboundNatProvider;
  81     private volatile OutboundNatProvider outboundNatProvider;
  82     private volatile ArpProvider arpProvider;
  83     private volatile RoutingProvider routingProvider;
  84
  85     private Set<String> inboundIpRewriteCache;
  86     private Set<String> outboundIpRewriteCache;
  87     private Set<String> inboundIpRewriteExclusionCache;
  88     private Set<String> outboundIpRewriteExclusionCache;
  89     private Set<String> routerInterfacesCache;
  90     private Set<String> staticArpEntryCache;
  91     private Set<String> l3ForwardingCache;
  92     private Set<String> defaultRouteCache;
  93     private Map<String, String> networkIdToRouterMacCache;
  94     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  95     private Boolean enabled = false;
  96
  97     void init() {
  98         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  99         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
 100             this.inboundIpRewriteCache = new HashSet<>();
 101             this.outboundIpRewriteCache = new HashSet<>();
 102             this.inboundIpRewriteExclusionCache = new HashSet<>();
 103             this.outboundIpRewriteExclusionCache = new HashSet<>();
 104             this.routerInterfacesCache = new HashSet<>();
 105             this.staticArpEntryCache = new HashSet<>();
 106             this.l3ForwardingCache = new HashSet<>();
 107             this.defaultRouteCache = new HashSet<>();
 108             this.networkIdToRouterMacCache = new HashMap<>();
 109             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 110
 111             this.enabled = true;
 112             logger.info("OVSDB L3 forwarding is enabled");
 113         } else {
 114             logger.debug("OVSDB L3 forwarding is disabled");
 115         }
 116     }
 117
 118     //
 119     // Callbacks from OVSDB's northbound handlers
 120     //
 121
 122     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 123         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 124         if (!this.enabled)
 125             return;
 126     }
 127
 128     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 129         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 130         if (!this.enabled)
 131             return;
 132
 133         final boolean isDelete = action == Action.DELETE;
 134
 135         // Treat the port event as a router interface event if the port belongs to router. This is a
 136         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 137         //
 138         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 139             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 140                 NeutronRouter_Interface neutronRouterInterface =
 141                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 142                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 143                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 144
 145                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 146             }
 147         } else {
 148             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 149             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 150             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 151             // there.
 152             //
 153             if (!isDelete) {
 154                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 155                     NeutronRouter_Interface neutronRouterInterface =
 156                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 157                     if (neutronRouterInterface != null) {
 158                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 159                     }
 160                 }
 161             }
 162             this.updateL3ForNeutronPort(neutronPort, isDelete);
 163         }
 164     }
 165
 166     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 167         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 168         if (!this.enabled)
 169             return;
 170     }
 171
 172     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 173                                                   final NeutronRouter_Interface neutronRouterInterface,
 174                                                   Action action) {
 175         logger.debug("Router interface {} got event {}. Subnet {}",
 176                      neutronRouterInterface.getPortUUID(),
 177                      action,
 178                      neutronRouterInterface.getSubnetUUID());
 179         if (!this.enabled)
 180             return;
 181
 182         final boolean isDelete = action == Action.DELETE;
 183
 184         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 185
 186         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 187         // see if they are affected by l3
 188         //
 189         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 190             boolean currPortShouldBeDeleted = false;
 191             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 192             if (isDelete) {
 193                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 194                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 195                         currPortShouldBeDeleted = true;
 196                         break;
 197                     }
 198                 }
 199             }
 200             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 201         }
 202     }
 203
 204     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 205                                              Action action) {
 206         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 207                      neutronFloatingIP.getFixedIPAddress(),
 208                      neutronFloatingIP.getFloatingIPAddress(),
 209                      neutronFloatingIP.getFloatingNetworkUUID());
 210         if (!this.enabled)
 211             return;
 212
 213         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 214     }
 215
 216     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 217         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 218         if (!this.enabled)
 219             return;
 220     }
 221
 222     //
 223     // Callbacks from OVSDB's southbound handler
 224     //
 225     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 226                                      Action action) {
 227         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 228                      action, node, intf.getName(), neutronNetwork);
 229         if (!this.enabled)
 230             return;
 231
 232         // See if there is an external uuid, so we can find the respective neutronPort
 233         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 234         if (externalIds == null) {
 235             return;
 236         }
 237         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 238         if (neutronPortId == null) {
 239             return;
 240         }
 241         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 242         if (neutronPort == null) {
 243             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 244                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 245             return;
 246         }
 247         this.handleNeutronPortEvent(neutronPort, action);
 248     }
 249
 250     //
 251     // Internal helpers
 252     //
 253     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 254
 255         final String networkUUID = neutronPort.getNetworkUUID();
 256         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 257
 258         // If there is no router interface handling the networkUUID, we are done
 259         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 260             return;
 261         }
 262
 263         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 264         // router interface are handled via handleNeutronRouterInterfaceEvent.
 265         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 266             return;
 267         }
 268
 269         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 270         final String providerSegmentationId = neutronNetwork != null ?
 271                                               neutronNetwork.getProviderSegmentationID() : null;
 272         final String tenantMac = neutronPort.getMacAddress();
 273
 274         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 275             tenantMac == null || tenantMac.isEmpty()) {
 276             return;  // done: go no further w/out all the info needed...
 277         }
 278
 279         final Action action = isDelete ? Action.DELETE : Action.ADD;
 280         List<Node> nodes = connectionService.getNodes();
 281         if (nodes.isEmpty()) {
 282             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 283         }
 284         for (Node node : nodes) {
 285             final Long dpid = getDpid(node);
 286             final boolean tenantNetworkPresentInNode =
 287                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 288             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 289                 final String tenantIpStr = neutronIP.getIpAddress();
 290                 if (tenantIpStr.isEmpty()) {
 291                     continue;
 292                 }
 293
 294                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 295                 // still needed when routing to subnets non-local to node (bug 2076).
 296                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 297
 298                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 299                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 300                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 301             }
 302         }
 303     }
 304
 305     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 306                                            String macAddress, String ipStr,
 307                                            Action actionForNode) {
 308         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 309         // will look at desired action for node.
 310         //
 311         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 312         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 313
 314         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 315             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 316                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 317             return;
 318         }
 319         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 320             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 321                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 322             return;
 323         }
 324
 325         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 326                                                        macAddress, ipStr, actionForNode);
 327         if (status.isSuccess()) {
 328             // Update cache
 329             if (actionForNode == Action.ADD) {
 330                 l3ForwardingCache.add(cacheKey);
 331             } else {
 332                 l3ForwardingCache.remove(cacheKey);
 333             }
 334         }
 335     }
 336
 337     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 338                                              String macAddress,
 339                                              String address,
 340                                              Action actionForNode) {
 341         Status status;
 342         try {
 343             InetAddress inetAddress = InetAddress.getByName(address);
 344             status = l3ForwardingProvider == null ?
 345                      new Status(StatusCode.SUCCESS) :
 346                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 347                                                                       inetAddress, macAddress, actionForNode);
 348         } catch (UnknownHostException e) {
 349             status = new Status(StatusCode.BADREQUEST);
 350         }
 351
 352         if (status.isSuccess()) {
 353             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 354                          l3ForwardingProvider == null ? "skipped" : "programmed",
 355                          macAddress, address, node, actionForNode);
 356         } else {
 357             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 358                          macAddress, address, node, actionForNode, status);
 359         }
 360         return status;
 361     }
 362
 363     // --
 364
 365     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 366                                                        Boolean isDelete) {
 367         Preconditions.checkNotNull(destNeutronRouterInterface);
 368
 369         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 370         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 371         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 372         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 373         final NeutronNetwork neutronNetwork = subnet != null ?
 374                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 375         final String destinationSegmentationId = neutronNetwork != null ?
 376                                                  neutronNetwork.getProviderSegmentationID() : null;
 377         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 378         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 379         final String cidr = subnet != null ? subnet.getCidr() : null;
 380         final int mask = getMaskLenFromCidr(cidr);
 381
 382         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 383                      destNeutronRouterInterface, isDelete);
 384
 385         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 386             cidr == null || cidr.isEmpty() ||
 387             macAddress == null || macAddress.isEmpty() ||
 388             ipList == null || ipList.isEmpty()) {
 389             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 390                          destinationSegmentationId, cidr, macAddress, ipList);
 391             return;  // done: go no further w/out all the info needed...
 392         }
 393
 394         final Action action = isDelete ? Action.DELETE : Action.ADD;
 395
 396         // Keep cache for finding router's mac from network uuid -- add
 397         //
 398         if (! isDelete) {
 399             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 400             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 401         }
 402
 403         List<Node> nodes = connectionService.getNodes();
 404         if (nodes.isEmpty()) {
 405             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 406         }
 407         for (Node node : nodes) {
 408             final Long dpid = getDpid(node);
 409             final Action actionForNode =
 410                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 411                     action : Action.DELETE;
 412
 413             for (Neutron_IPs neutronIP : ipList) {
 414                 final String ipStr = neutronIP.getIpAddress();
 415                 if (ipStr.isEmpty()) {
 416                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 417                                  node.getID(), ipStr);
 418                     continue;
 419                 }
 420
 421                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 422                 //
 423                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 424                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 425                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 426                                                               neutronNetwork, destinationSegmentationId,
 427                                                               macAddress, ipStr, mask, actionForNode,
 428                                                               true /*isReflexsive*/);
 429                 }
 430
 431                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 432             }
 433
 434             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 435             // for the external neutron networks.
 436             //
 437             {
 438                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 439                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 440                                                 cidr, actionForRewriteExclusion);
 441                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 442                                                 cidr, actionForRewriteExclusion);
 443             }
 444
 445             // Default route. For non-external subnet, make sure that there is none configured.
 446             //
 447             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 448                 final Action actionForNodeDefaultRoute =
 449                         isExternal ? actionForNode : Action.DELETE;
 450                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 451                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 452                                           actionForNodeDefaultRoute);
 453             }
 454         }
 455
 456         // Keep cache for finding router's mac from network uuid -- remove
 457         //
 458         if (isDelete) {
 459             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 460             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 461         }
 462     }
 463
 464     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 465                                                            final Long dpid,
 466                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 467                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 468                                                            final NeutronNetwork dstNeutronNetwork,
 469                                                            final String destinationSegmentationId,
 470                                                            final String dstMacAddress,
 471                                                            final String destIpStr,
 472                                                            final int destMask,
 473                                                            final Action actionForNode,
 474                                                            Boolean isReflexsive) {
 475         Preconditions.checkNotNull(srcNeutronRouterInterface);
 476         Preconditions.checkNotNull(dstNeutronRouterInterface);
 477
 478         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 479         if (sourceSubnetId == null) {
 480             logger.error("Could not get provider Subnet ID from router interface {}",
 481                          srcNeutronRouterInterface.getID());
 482             return;
 483         }
 484
 485         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 486         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 487         if (sourceNetworkId == null) {
 488             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 489             return;
 490         }
 491
 492         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 493         if (sourceNetwork == null) {
 494             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 495             return;
 496         }
 497
 498         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 499             // Isolate subnets from different tenants within the same router
 500             return;
 501         }
 502         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 503         if (sourceSegmentationId == null) {
 504             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 505             return;
 506         }
 507         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 508             // Skip 'self'
 509             return;
 510         }
 511
 512         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 513                                      dstMacAddress, destIpStr, destMask, actionForNode);
 514
 515         // Flip roles src->dst; dst->src
 516         if (isReflexsive) {
 517             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 518             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 519             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 520             final String cidr2 = sourceSubnet.getCidr();
 521             final int mask2 = getMaskLenFromCidr(cidr2);
 522
 523             if (cidr2 == null || cidr2.isEmpty() ||
 524                 macAddress2 == null || macAddress2.isEmpty() ||
 525                 ipList2 == null || ipList2.isEmpty()) {
 526                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 527                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 528                 return;  // done: go no further w/out all the info needed...
 529             }
 530
 531             for (Neutron_IPs neutronIP2 : ipList2) {
 532                 final String ipStr2 = neutronIP2.getIpAddress();
 533                 if (ipStr2.isEmpty()) {
 534                     continue;
 535                 }
 536                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 537                                                           srcNeutronRouterInterface,
 538                                                           sourceNetwork, sourceSegmentationId,
 539                                                           macAddress2, ipStr2, mask2, actionForNode,
 540                                                           false /*isReflexsive*/);
 541             }
 542         }
 543     }
 544
 545     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 546                                               String destinationSegmentationId,
 547                                               String macAddress, String ipStr, int mask,
 548                                               Action actionForNode) {
 549         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 550         // will look at desired action for node.
 551         //
 552         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 553                                 ipStr + "/" + Integer.toString(mask);
 554         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 555
 556         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 557             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 558                          "action {} is already done",
 559                          node.getNodeIDString(), sourceSegmentationId, destinationSegmentationId,
 560                          ipStr, mask, actionForNode);
 561             return;
 562         }
 563         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 564             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 565                          "action {} is already done",
 566                          node.getNodeIDString(), sourceSegmentationId, destinationSegmentationId,
 567                          ipStr, mask, actionForNode);
 568             return;
 569         }
 570
 571         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 572                                                           macAddress, ipStr, mask, actionForNode);
 573         if (status.isSuccess()) {
 574             // Update cache
 575             if (actionForNode == Action.ADD) {
 576                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 577                 routerInterfacesCache.add(cacheKey);
 578             } else {
 579                 // TODO: multiTenantAwareRouter.removeInterface(...);
 580                 routerInterfacesCache.remove(cacheKey);
 581             }
 582         }
 583     }
 584
 585     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 586                                                 String destinationSegmentationId,
 587                                                 String macAddress,
 588                                                 String address, int mask,
 589                                                 Action actionForNode) {
 590         Status status;
 591         try {
 592             InetAddress inetAddress = InetAddress.getByName(address);
 593             status = routingProvider == null ?
 594                      new Status(StatusCode.SUCCESS) :
 595                      routingProvider.programRouterInterface(node, dpid, sourceSegmentationId, destinationSegmentationId,
 596                                                             macAddress, inetAddress, mask, actionForNode);
 597         } catch (UnknownHostException e) {
 598             status = new Status(StatusCode.BADREQUEST);
 599         }
 600
 601         if (status.isSuccess()) {
 602             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 603                          routingProvider == null ? "skipped" : "programmed",
 604                          macAddress, address, mask, node, actionForNode);
 605         } else {
 606             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 607                          macAddress, address, mask, node, actionForNode, status);
 608         }
 609         return status;
 610     }
 611
 612     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 613                                         String macAddress, String ipStr,
 614                                         Action actionForNode) {
 615         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 616         // will look at desired action for node.
 617         //
 618         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 619         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 620
 621         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 622             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 623                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 624             return;
 625         }
 626         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 627             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 628                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 629             return;
 630         }
 631
 632         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 633                                                     macAddress, ipStr, actionForNode);
 634         if (status.isSuccess()) {
 635             // Update cache
 636             if (actionForNode == Action.ADD) {
 637                 staticArpEntryCache.add(cacheKey);
 638             } else {
 639                 staticArpEntryCache.remove(cacheKey);
 640             }
 641         }
 642     }
 643
 644     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 645                                                 String macAddress,
 646                                                 String address,
 647                                                 Action actionForNode) {
 648         Status status;
 649         try {
 650             InetAddress inetAddress = InetAddress.getByName(address);
 651             status = arpProvider == null ?
 652                      new Status(StatusCode.SUCCESS) :
 653                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 654                                                        macAddress, inetAddress, actionForNode);
 655         } catch (UnknownHostException e) {
 656             status = new Status(StatusCode.BADREQUEST);
 657         }
 658
 659         if (status.isSuccess()) {
 660             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 661                          arpProvider == null ? "skipped" : "programmed",
 662                          macAddress, address, node, actionForNode);
 663         } else {
 664             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 665                          macAddress, address, node, actionForNode, status);
 666         }
 667         return status;
 668     }
 669
 670     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 671                                                  final boolean isInbound, String cidr,
 672                                                  Action actionForRewriteExclusion) {
 673         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 674         // will look at desired action for node.
 675         //
 676         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 677         final Boolean isProgrammed = isInbound ?
 678                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 679                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 680
 681         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 682             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 683                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 684                          actionForRewriteExclusion);
 685             return;
 686         }
 687         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 688             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 689                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 690                          actionForRewriteExclusion);
 691             return;
 692         }
 693
 694         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 695                                                              isInbound, actionForRewriteExclusion);
 696         if (status.isSuccess()) {
 697             // Update cache
 698             if (actionForRewriteExclusion == Action.ADD) {
 699                 if (isInbound) {
 700                     inboundIpRewriteExclusionCache.add(cacheKey);
 701                 } else {
 702                     outboundIpRewriteExclusionCache.add(cacheKey);
 703                 }
 704             } else {
 705                 if (isInbound) {
 706                     inboundIpRewriteExclusionCache.remove(cacheKey);
 707                 } else {
 708                     outboundIpRewriteExclusionCache.remove(cacheKey);
 709                 }
 710             }
 711         }
 712     }
 713
 714     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 715                                                    final boolean isInbound, Action actionForNode) {
 716         Status status;
 717         if (isInbound) {
 718             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 719                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 720                                                                   actionForNode);
 721         } else {
 722             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 723                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 724                                                                    actionForNode);
 725         }
 726
 727         if (status.isSuccess()) {
 728             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 729             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 730                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 731                          cidr, node, actionForNode);
 732         } else {
 733             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 734                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 735         }
 736         return status;
 737     }
 738
 739     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 740                                            String defaultGatewayMacAddress, String gatewayIp,
 741                                            Action actionForNodeDefaultRoute) {
 742         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 743         // will look at desired action for node.
 744         //
 745         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 746         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 747
 748         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 749             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 750                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 751                          actionForNodeDefaultRoute);
 752             return;
 753         }
 754         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 755             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 756                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 757                          actionForNodeDefaultRoute);
 758             return;
 759         }
 760
 761         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 762                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 763         if (status.isSuccess()) {
 764             // Update cache
 765             if (actionForNodeDefaultRoute == Action.ADD) {
 766                 defaultRouteCache.add(cacheKey);
 767             } else {
 768                 defaultRouteCache.remove(cacheKey);
 769             }
 770         }
 771     }
 772
 773     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 774                                           String defaultGatewayMacAddress,
 775                                           String gatewayIp,
 776                                           Action actionForNodeDefaultRoute) {
 777         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 778         if (defaultGatewayMacAddress == null) {
 779             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 780                          gatewayIp, node, actionForNodeDefaultRoute);
 781             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 782         }
 783
 784         Status status;
 785         try {
 786             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 787             status = routingProvider == null ?
 788                      new Status(StatusCode.SUCCESS) :
 789                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 790                                                               defaultGatewayMacAddress, inetAddress,
 791                                                               actionForNodeDefaultRoute);
 792         } catch (UnknownHostException e) {
 793             status = new Status(StatusCode.BADREQUEST);
 794         }
 795
 796         if (status.isSuccess()) {
 797             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 798                          routingProvider == null ? "skipped" : "programmed",
 799                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 800         } else {
 801             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 802                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 803         }
 804         return status;
 805     }
 806
 807     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 808         Preconditions.checkNotNull(neutronFloatingIP);
 809
 810         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 811         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 812
 813         // If there is no router interface handling the networkUUID, we are done
 814         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 815             return;
 816         }
 817
 818         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 819         final String providerSegmentationId = neutronNetwork != null ?
 820                                               neutronNetwork.getProviderSegmentationID() : null;
 821         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 822         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 823
 824         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 825             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 826             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 827             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 828             return;  // done: go no further w/out all the info needed...
 829         }
 830
 831         final Action action = isDelete ? Action.DELETE : Action.ADD;
 832         List<Node> nodes = connectionService.getNodes();
 833         if (nodes.isEmpty()) {
 834             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 835         }
 836         for (Node node : nodes) {
 837             final Long dpid = getDpid(node);
 838             final Action actionForNode =
 839                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 840                     action : Action.DELETE;
 841
 842             // Rewrite from float to fixed and vice-versa
 843             //
 844             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 845                                    floatingIpAddress, fixedIPAddress, actionForNode);
 846             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 847                                    fixedIPAddress, floatingIpAddress, actionForNode);
 848
 849             // Respond to arps for the floating ip address
 850             //
 851             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 852                                    actionForNode);
 853         }
 854     }
 855
 856     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 857                                         final boolean isInbound,
 858                                         String matchAddress, String rewriteAddress,
 859                                         Action actionForNode) {
 860         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 861         // will look at desired action for node.
 862         //
 863         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 864                                 matchAddress + ":" + rewriteAddress;
 865         final Boolean isProgrammed = isInbound ?
 866                                      inboundIpRewriteCache.contains(cacheKey) :
 867                                      outboundIpRewriteCache.contains(cacheKey);
 868
 869         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 870             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 871                          " is already done",
 872                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 873                          matchAddress, rewriteAddress, actionForNode);
 874             return;
 875         }
 876         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 877             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 878                          " is already done",
 879                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 880                          matchAddress, rewriteAddress, actionForNode);
 881             return;
 882         }
 883
 884         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 885                                                     matchAddress, rewriteAddress, actionForNode);
 886         if (status.isSuccess()) {
 887             // Update cache
 888             if (actionForNode == Action.ADD) {
 889                 if (isInbound) {
 890                     inboundIpRewriteCache.add(cacheKey);
 891                 } else {
 892                     outboundIpRewriteCache.add(cacheKey);
 893                 }
 894             } else {
 895                 if (isInbound) {
 896                     inboundIpRewriteCache.remove(cacheKey);
 897                 } else {
 898                     outboundIpRewriteCache.remove(cacheKey);
 899                 }
 900             }
 901         }
 902     }
 903
 904     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 905                                           final boolean isInbound,
 906                                           String matchAddress, String rewriteAddress,
 907                                           Action actionForNode) {
 908         Status status;
 909         try {
 910             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 911             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 912             if (isInbound) {
 913                 status = inboundNatProvider == null ?
 914                          new Status(StatusCode.SUCCESS) :
 915                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 916                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 917             } else {
 918                 status = outboundNatProvider == null ?
 919                          new Status(StatusCode.SUCCESS) :
 920                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 921                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 922             }
 923         } catch (UnknownHostException e) {
 924             status = new Status(StatusCode.BADREQUEST);
 925         }
 926
 927         if (status.isSuccess()) {
 928             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 929             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 930                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 931                          matchAddress, rewriteAddress, node, actionForNode);
 932         } else {
 933             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 934                          (isInbound ? "inbound" : "outbound"),
 935                          matchAddress, rewriteAddress, node, actionForNode, status);
 936         }
 937         return status;
 938     }
 939
 940     //
 941     // More Internals
 942     //
 943
 944     private int getMaskLenFromCidr(String cidr) {
 945         if (cidr == null) return 0;
 946         String[] splits = cidr.split("/");
 947         if (splits.length != 2) return 0;
 948
 949         int result;
 950         try {
 951             result = Integer.parseInt(splits[1].trim());
 952         }
 953         catch (NumberFormatException nfe)
 954         {
 955             result = 0;
 956         }
 957         return result;
 958     }
 959
 960     private Long getDpid (Node node) {
 961         Preconditions.checkNotNull(ovsdbConfigurationService);
 962
 963         String bridgeName = configurationService.getIntegrationBridgeName();
 964         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 965         if (bridgeUuid == null) {
 966             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 967             return 0L;
 968         }
 969
 970         try {
 971             Row bridgeRow =  ovsdbConfigurationService
 972                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 973             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 974             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 975             if (dpids == null || dpids.size() == 0) return 0L;
 976             return HexEncode.stringToLong((String) dpids.toArray()[0]);
 977         } catch (Exception e) {
 978             logger.error("Error finding Bridge's OF DPID", e);
 979             return 0L;
 980         }
 981     }
 982
 983     private String getInternalBridgeUUID (Node node, String bridgeName) {
 984         Preconditions.checkNotNull(ovsdbConfigurationService);
 985         try {
 986             Map<String, Row> bridgeTable =
 987                     ovsdbConfigurationService.getRows(node,
 988                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 989             if (bridgeTable == null) return null;
 990             for (String key : bridgeTable.keySet()) {
 991                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 992                 if (bridge.getName().equals(bridgeName)) return key;
 993             }
 994         } catch (Exception e) {
 995             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 996         }
 997         return null;
 998     }
 999 }
1000
1001