openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.lib.notation.Row;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  27 import org.opendaylight.ovsdb.plugin.api.OvsdbConfigurationService;
  28 import org.opendaylight.ovsdb.plugin.api.OvsdbConnectionService;
  29 import org.opendaylight.ovsdb.plugin.api.Status;
  30 import org.opendaylight.ovsdb.plugin.api.StatusCode;
  31 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  38 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  39 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  40 import org.opendaylight.ovsdb.utils.mdsal.node.StringConvertor;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  42
  43 import com.google.common.base.Preconditions;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import java.net.InetAddress;
  48 import java.net.UnknownHostException;
  49 import java.util.HashMap;
  50 import java.util.HashSet;
  51 import java.util.List;
  52 import java.util.Map;
  53 import java.util.Set;
  54
  55 /**
  56  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  57  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  58  * as well as the multi-tenant router forwarding provider.
  59  */
  60 public class NeutronL3Adapter {
  61
  62     /**
  63      * Logger instance.
  64      */
  65     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  66
  67     // The implementation for each of these services is resolved by the OSGi Service Manager
  68     private volatile ConfigurationService configurationService;
  69     private volatile TenantNetworkManager tenantNetworkManager;
  70     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  71     private volatile OvsdbConnectionService connectionService;
  72     private volatile INeutronNetworkCRUD neutronNetworkCache;
  73     private volatile INeutronSubnetCRUD neutronSubnetCache;
  74     private volatile INeutronPortCRUD neutronPortCache;
  75     private volatile L3ForwardingProvider l3ForwardingProvider;
  76     private volatile InboundNatProvider inboundNatProvider;
  77     private volatile OutboundNatProvider outboundNatProvider;
  78     private volatile ArpProvider arpProvider;
  79     private volatile RoutingProvider routingProvider;
  80
  81     private Set<String> inboundIpRewriteCache;
  82     private Set<String> outboundIpRewriteCache;
  83     private Set<String> inboundIpRewriteExclusionCache;
  84     private Set<String> outboundIpRewriteExclusionCache;
  85     private Set<String> routerInterfacesCache;
  86     private Set<String> staticArpEntryCache;
  87     private Set<String> l3ForwardingCache;
  88     private Set<String> defaultRouteCache;
  89     private Map<String, String> networkIdToRouterMacCache;
  90     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  91     private Boolean enabled = false;
  92
  93     void init() {
  94         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  95         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  96             this.inboundIpRewriteCache = new HashSet<>();
  97             this.outboundIpRewriteCache = new HashSet<>();
  98             this.inboundIpRewriteExclusionCache = new HashSet<>();
  99             this.outboundIpRewriteExclusionCache = new HashSet<>();
 100             this.routerInterfacesCache = new HashSet<>();
 101             this.staticArpEntryCache = new HashSet<>();
 102             this.l3ForwardingCache = new HashSet<>();
 103             this.defaultRouteCache = new HashSet<>();
 104             this.networkIdToRouterMacCache = new HashMap<>();
 105             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 106
 107             this.enabled = true;
 108             logger.info("OVSDB L3 forwarding is enabled");
 109         } else {
 110             logger.debug("OVSDB L3 forwarding is disabled");
 111         }
 112     }
 113
 114     //
 115     // Callbacks from OVSDB's northbound handlers
 116     //
 117
 118     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 119         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 120         if (!this.enabled)
 121             return;
 122     }
 123
 124     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 125         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 126         if (!this.enabled)
 127             return;
 128
 129         final boolean isDelete = action == Action.DELETE;
 130
 131         // Treat the port event as a router interface event if the port belongs to router. This is a
 132         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 133         //
 134         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 135             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 136                 NeutronRouter_Interface neutronRouterInterface =
 137                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 138                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 139                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 140
 141                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 142             }
 143         } else {
 144             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 145             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 146             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 147             // there.
 148             //
 149             if (!isDelete) {
 150                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 151                     NeutronRouter_Interface neutronRouterInterface =
 152                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 153                     if (neutronRouterInterface != null) {
 154                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 155                     }
 156                 }
 157             }
 158             this.updateL3ForNeutronPort(neutronPort, isDelete);
 159         }
 160     }
 161
 162     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 163         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 164         if (!this.enabled)
 165             return;
 166     }
 167
 168     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 169                                                   final NeutronRouter_Interface neutronRouterInterface,
 170                                                   Action action) {
 171         logger.debug("Router interface {} got event {}. Subnet {}",
 172                      neutronRouterInterface.getPortUUID(),
 173                      action,
 174                      neutronRouterInterface.getSubnetUUID());
 175         if (!this.enabled)
 176             return;
 177
 178         final boolean isDelete = action == Action.DELETE;
 179
 180         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 181
 182         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 183         // see if they are affected by l3
 184         //
 185         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 186             boolean currPortShouldBeDeleted = false;
 187             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 188             if (isDelete) {
 189                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 190                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 191                         currPortShouldBeDeleted = true;
 192                         break;
 193                     }
 194                 }
 195             }
 196             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 197         }
 198     }
 199
 200     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 201                                              Action action) {
 202         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 203                      neutronFloatingIP.getFixedIPAddress(),
 204                      neutronFloatingIP.getFloatingIPAddress(),
 205                      neutronFloatingIP.getFloatingNetworkUUID());
 206         if (!this.enabled)
 207             return;
 208
 209         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 210     }
 211
 212     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 213         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 214         if (!this.enabled)
 215             return;
 216     }
 217
 218     //
 219     // Callbacks from OVSDB's southbound handler
 220     //
 221     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 222                                      Action action) {
 223         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 224                      action, node, intf.getName(), neutronNetwork);
 225         if (!this.enabled)
 226             return;
 227
 228         // See if there is an external uuid, so we can find the respective neutronPort
 229         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 230         if (externalIds == null) {
 231             return;
 232         }
 233         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 234         if (neutronPortId == null) {
 235             return;
 236         }
 237         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 238         if (neutronPort == null) {
 239             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 240                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 241             return;
 242         }
 243         this.handleNeutronPortEvent(neutronPort, action);
 244     }
 245
 246     //
 247     // Internal helpers
 248     //
 249     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 250
 251         final String networkUUID = neutronPort.getNetworkUUID();
 252         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 253
 254         // If there is no router interface handling the networkUUID, we are done
 255         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 256             return;
 257         }
 258
 259         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 260         // router interface are handled via handleNeutronRouterInterfaceEvent.
 261         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 262             return;
 263         }
 264
 265         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 266         final String providerSegmentationId = neutronNetwork != null ?
 267                                               neutronNetwork.getProviderSegmentationID() : null;
 268         final String tenantMac = neutronPort.getMacAddress();
 269
 270         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 271             tenantMac == null || tenantMac.isEmpty()) {
 272             return;  // done: go no further w/out all the info needed...
 273         }
 274
 275         final Action action = isDelete ? Action.DELETE : Action.ADD;
 276         List<Node> nodes = connectionService.getNodes();
 277         if (nodes.isEmpty()) {
 278             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 279         }
 280         for (Node node : nodes) {
 281             final Long dpid = getDpid(node);
 282             final boolean tenantNetworkPresentInNode =
 283                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 284             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 285                 final String tenantIpStr = neutronIP.getIpAddress();
 286                 if (tenantIpStr.isEmpty()) {
 287                     continue;
 288                 }
 289
 290                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 291                 // still needed when routing to subnets non-local to node (bug 2076).
 292                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 293
 294                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 295                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 296                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 297             }
 298         }
 299     }
 300
 301     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 302                                            String macAddress, String ipStr,
 303                                            Action actionForNode) {
 304         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 305         // will look at desired action for node.
 306         //
 307         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 308         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 309
 310         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 311             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 312                          node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 313             return;
 314         }
 315         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 316             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 317                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 318             return;
 319         }
 320
 321         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 322                                                        macAddress, ipStr, actionForNode);
 323         if (status.isSuccess()) {
 324             // Update cache
 325             if (actionForNode == Action.ADD) {
 326                 l3ForwardingCache.add(cacheKey);
 327             } else {
 328                 l3ForwardingCache.remove(cacheKey);
 329             }
 330         }
 331     }
 332
 333     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 334                                              String macAddress,
 335                                              String address,
 336                                              Action actionForNode) {
 337         Status status;
 338         try {
 339             InetAddress inetAddress = InetAddress.getByName(address);
 340             status = l3ForwardingProvider == null ?
 341                      new Status(StatusCode.SUCCESS) :
 342                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 343                                                                       inetAddress, macAddress, actionForNode);
 344         } catch (UnknownHostException e) {
 345             status = new Status(StatusCode.BADREQUEST);
 346         }
 347
 348         if (status.isSuccess()) {
 349             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 350                          l3ForwardingProvider == null ? "skipped" : "programmed",
 351                          macAddress, address, node, actionForNode);
 352         } else {
 353             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 354                          macAddress, address, node, actionForNode, status);
 355         }
 356         return status;
 357     }
 358
 359     // --
 360
 361     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 362                                                        Boolean isDelete) {
 363         Preconditions.checkNotNull(destNeutronRouterInterface);
 364
 365         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 366         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 367         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 368         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 369         final NeutronNetwork neutronNetwork = subnet != null ?
 370                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 371         final String destinationSegmentationId = neutronNetwork != null ?
 372                                                  neutronNetwork.getProviderSegmentationID() : null;
 373         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 374         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 375         final String cidr = subnet != null ? subnet.getCidr() : null;
 376         final int mask = getMaskLenFromCidr(cidr);
 377
 378         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 379                      destNeutronRouterInterface, isDelete);
 380
 381         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 382             cidr == null || cidr.isEmpty() ||
 383             macAddress == null || macAddress.isEmpty() ||
 384             ipList == null || ipList.isEmpty()) {
 385             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 386                          destinationSegmentationId, cidr, macAddress, ipList);
 387             return;  // done: go no further w/out all the info needed...
 388         }
 389
 390         final Action action = isDelete ? Action.DELETE : Action.ADD;
 391
 392         // Keep cache for finding router's mac from network uuid -- add
 393         //
 394         if (! isDelete) {
 395             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 396             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 397         }
 398
 399         List<Node> nodes = connectionService.getNodes();
 400         if (nodes.isEmpty()) {
 401             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 402         }
 403         for (Node node : nodes) {
 404             final Long dpid = getDpid(node);
 405             final Action actionForNode =
 406                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 407                     action : Action.DELETE;
 408
 409             for (Neutron_IPs neutronIP : ipList) {
 410                 final String ipStr = neutronIP.getIpAddress();
 411                 if (ipStr.isEmpty()) {
 412                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 413                             node.getId().getValue(), ipStr);
 414                     continue;
 415                 }
 416
 417                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 418                 //
 419                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 420                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 421                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 422                                                               neutronNetwork, destinationSegmentationId,
 423                                                               macAddress, ipStr, mask, actionForNode,
 424                                                               true /*isReflexsive*/);
 425                 }
 426
 427                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 428             }
 429
 430             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 431             // for the external neutron networks.
 432             //
 433             {
 434                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 435                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 436                                                 cidr, actionForRewriteExclusion);
 437                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 438                                                 cidr, actionForRewriteExclusion);
 439             }
 440
 441             // Default route. For non-external subnet, make sure that there is none configured.
 442             //
 443             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 444                 final Action actionForNodeDefaultRoute =
 445                         isExternal ? actionForNode : Action.DELETE;
 446                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 447                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 448                                           actionForNodeDefaultRoute);
 449             }
 450         }
 451
 452         // Keep cache for finding router's mac from network uuid -- remove
 453         //
 454         if (isDelete) {
 455             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 456             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 457         }
 458     }
 459
 460     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 461                                                            final Long dpid,
 462                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 463                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 464                                                            final NeutronNetwork dstNeutronNetwork,
 465                                                            final String destinationSegmentationId,
 466                                                            final String dstMacAddress,
 467                                                            final String destIpStr,
 468                                                            final int destMask,
 469                                                            final Action actionForNode,
 470                                                            Boolean isReflexsive) {
 471         Preconditions.checkNotNull(srcNeutronRouterInterface);
 472         Preconditions.checkNotNull(dstNeutronRouterInterface);
 473
 474         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 475         if (sourceSubnetId == null) {
 476             logger.error("Could not get provider Subnet ID from router interface {}",
 477                          srcNeutronRouterInterface.getID());
 478             return;
 479         }
 480
 481         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 482         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 483         if (sourceNetworkId == null) {
 484             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 485             return;
 486         }
 487
 488         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 489         if (sourceNetwork == null) {
 490             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 491             return;
 492         }
 493
 494         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 495             // Isolate subnets from different tenants within the same router
 496             return;
 497         }
 498         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 499         if (sourceSegmentationId == null) {
 500             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 501             return;
 502         }
 503         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 504             // Skip 'self'
 505             return;
 506         }
 507
 508         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 509                                      dstMacAddress, destIpStr, destMask, actionForNode);
 510
 511         // Flip roles src->dst; dst->src
 512         if (isReflexsive) {
 513             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 514             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 515             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 516             final String cidr2 = sourceSubnet.getCidr();
 517             final int mask2 = getMaskLenFromCidr(cidr2);
 518
 519             if (cidr2 == null || cidr2.isEmpty() ||
 520                 macAddress2 == null || macAddress2.isEmpty() ||
 521                 ipList2 == null || ipList2.isEmpty()) {
 522                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 523                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 524                 return;  // done: go no further w/out all the info needed...
 525             }
 526
 527             for (Neutron_IPs neutronIP2 : ipList2) {
 528                 final String ipStr2 = neutronIP2.getIpAddress();
 529                 if (ipStr2.isEmpty()) {
 530                     continue;
 531                 }
 532                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 533                                                           srcNeutronRouterInterface,
 534                                                           sourceNetwork, sourceSegmentationId,
 535                                                           macAddress2, ipStr2, mask2, actionForNode,
 536                                                           false /*isReflexsive*/);
 537             }
 538         }
 539     }
 540
 541     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 542                                               String destinationSegmentationId,
 543                                               String macAddress, String ipStr, int mask,
 544                                               Action actionForNode) {
 545         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 546         // will look at desired action for node.
 547         //
 548         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 549                                 ipStr + "/" + Integer.toString(mask);
 550         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 551
 552         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 553             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 554                          "action {} is already done",
 555                          node.getId().getValue(), sourceSegmentationId, destinationSegmentationId,
 556                          ipStr, mask, actionForNode);
 557             return;
 558         }
 559         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 560             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 561                          "action {} is already done",
 562                          node.getId().getValue(), sourceSegmentationId, destinationSegmentationId,
 563                          ipStr, mask, actionForNode);
 564             return;
 565         }
 566
 567         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 568                                                           macAddress, ipStr, mask, actionForNode);
 569         if (status.isSuccess()) {
 570             // Update cache
 571             if (actionForNode == Action.ADD) {
 572                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 573                 routerInterfacesCache.add(cacheKey);
 574             } else {
 575                 // TODO: multiTenantAwareRouter.removeInterface(...);
 576                 routerInterfacesCache.remove(cacheKey);
 577             }
 578         }
 579     }
 580
 581     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 582                                                 String destinationSegmentationId,
 583                                                 String macAddress,
 584                                                 String address, int mask,
 585                                                 Action actionForNode) {
 586         Status status;
 587         try {
 588             InetAddress inetAddress = InetAddress.getByName(address);
 589             status = routingProvider == null ?
 590                      new Status(StatusCode.SUCCESS) :
 591                      routingProvider.programRouterInterface(node, dpid, sourceSegmentationId, destinationSegmentationId,
 592                                                             macAddress, inetAddress, mask, actionForNode);
 593         } catch (UnknownHostException e) {
 594             status = new Status(StatusCode.BADREQUEST);
 595         }
 596
 597         if (status.isSuccess()) {
 598             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 599                          routingProvider == null ? "skipped" : "programmed",
 600                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 601                          actionForNode);
 602         } else {
 603             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 604                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 605                          actionForNode, status);
 606         }
 607         return status;
 608     }
 609
 610     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 611                                         String macAddress, String ipStr,
 612                                         Action actionForNode) {
 613         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 614         // will look at desired action for node.
 615         //
 616         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 617         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 618
 619         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 620             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 621                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 622             return;
 623         }
 624         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 625             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 626                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 627             return;
 628         }
 629
 630         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 631                                                     macAddress, ipStr, actionForNode);
 632         if (status.isSuccess()) {
 633             // Update cache
 634             if (actionForNode == Action.ADD) {
 635                 staticArpEntryCache.add(cacheKey);
 636             } else {
 637                 staticArpEntryCache.remove(cacheKey);
 638             }
 639         }
 640     }
 641
 642     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 643                                                 String macAddress,
 644                                                 String address,
 645                                                 Action actionForNode) {
 646         Status status;
 647         try {
 648             InetAddress inetAddress = InetAddress.getByName(address);
 649             status = arpProvider == null ?
 650                      new Status(StatusCode.SUCCESS) :
 651                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 652                                                        macAddress, inetAddress, actionForNode);
 653         } catch (UnknownHostException e) {
 654             status = new Status(StatusCode.BADREQUEST);
 655         }
 656
 657         if (status.isSuccess()) {
 658             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 659                          arpProvider == null ? "skipped" : "programmed",
 660                          macAddress, address, node, actionForNode);
 661         } else {
 662             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 663                          macAddress, address, node, actionForNode, status);
 664         }
 665         return status;
 666     }
 667
 668     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 669                                                  final boolean isInbound, String cidr,
 670                                                  Action actionForRewriteExclusion) {
 671         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 672         // will look at desired action for node.
 673         //
 674         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 675         final Boolean isProgrammed = isInbound ?
 676                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 677                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 678
 679         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 680             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 681                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 682                          actionForRewriteExclusion);
 683             return;
 684         }
 685         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 686             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 687                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 688                          actionForRewriteExclusion);
 689             return;
 690         }
 691
 692         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 693                                                              isInbound, actionForRewriteExclusion);
 694         if (status.isSuccess()) {
 695             // Update cache
 696             if (actionForRewriteExclusion == Action.ADD) {
 697                 if (isInbound) {
 698                     inboundIpRewriteExclusionCache.add(cacheKey);
 699                 } else {
 700                     outboundIpRewriteExclusionCache.add(cacheKey);
 701                 }
 702             } else {
 703                 if (isInbound) {
 704                     inboundIpRewriteExclusionCache.remove(cacheKey);
 705                 } else {
 706                     outboundIpRewriteExclusionCache.remove(cacheKey);
 707                 }
 708             }
 709         }
 710     }
 711
 712     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 713                                                    final boolean isInbound, Action actionForNode) {
 714         Status status;
 715         if (isInbound) {
 716             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 717                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 718                                                                   actionForNode);
 719         } else {
 720             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 721                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 722                                                                    actionForNode);
 723         }
 724
 725         if (status.isSuccess()) {
 726             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 727             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 728                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 729                          cidr, node, actionForNode);
 730         } else {
 731             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 732                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 733         }
 734         return status;
 735     }
 736
 737     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 738                                            String defaultGatewayMacAddress, String gatewayIp,
 739                                            Action actionForNodeDefaultRoute) {
 740         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 741         // will look at desired action for node.
 742         //
 743         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 744         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 745
 746         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 747             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 748                          node.getId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 749                          actionForNodeDefaultRoute);
 750             return;
 751         }
 752         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 753             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 754                          node.getId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 755                          actionForNodeDefaultRoute);
 756             return;
 757         }
 758
 759         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 760                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 761         if (status.isSuccess()) {
 762             // Update cache
 763             if (actionForNodeDefaultRoute == Action.ADD) {
 764                 defaultRouteCache.add(cacheKey);
 765             } else {
 766                 defaultRouteCache.remove(cacheKey);
 767             }
 768         }
 769     }
 770
 771     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 772                                           String defaultGatewayMacAddress,
 773                                           String gatewayIp,
 774                                           Action actionForNodeDefaultRoute) {
 775         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 776         if (defaultGatewayMacAddress == null) {
 777             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 778                          gatewayIp, node, actionForNodeDefaultRoute);
 779             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 780         }
 781
 782         Status status;
 783         try {
 784             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 785             status = routingProvider == null ?
 786                      new Status(StatusCode.SUCCESS) :
 787                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 788                                                               defaultGatewayMacAddress, inetAddress,
 789                                                               actionForNodeDefaultRoute);
 790         } catch (UnknownHostException e) {
 791             status = new Status(StatusCode.BADREQUEST);
 792         }
 793
 794         if (status.isSuccess()) {
 795             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 796                          routingProvider == null ? "skipped" : "programmed",
 797                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 798         } else {
 799             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 800                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 801         }
 802         return status;
 803     }
 804
 805     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 806         Preconditions.checkNotNull(neutronFloatingIP);
 807
 808         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 809         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 810
 811         // If there is no router interface handling the networkUUID, we are done
 812         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 813             return;
 814         }
 815
 816         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 817         final String providerSegmentationId = neutronNetwork != null ?
 818                                               neutronNetwork.getProviderSegmentationID() : null;
 819         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 820         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 821
 822         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 823             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 824             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 825             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 826             return;  // done: go no further w/out all the info needed...
 827         }
 828
 829         final Action action = isDelete ? Action.DELETE : Action.ADD;
 830         List<Node> nodes = connectionService.getNodes();
 831         if (nodes.isEmpty()) {
 832             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 833         }
 834         for (Node node : nodes) {
 835             final Long dpid = getDpid(node);
 836             final Action actionForNode =
 837                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 838                     action : Action.DELETE;
 839
 840             // Rewrite from float to fixed and vice-versa
 841             //
 842             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 843                                    floatingIpAddress, fixedIPAddress, actionForNode);
 844             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 845                                    fixedIPAddress, floatingIpAddress, actionForNode);
 846
 847             // Respond to arps for the floating ip address
 848             //
 849             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 850                                    actionForNode);
 851         }
 852     }
 853
 854     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 855                                         final boolean isInbound,
 856                                         String matchAddress, String rewriteAddress,
 857                                         Action actionForNode) {
 858         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 859         // will look at desired action for node.
 860         //
 861         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 862                                 matchAddress + ":" + rewriteAddress;
 863         final Boolean isProgrammed = isInbound ?
 864                                      inboundIpRewriteCache.contains(cacheKey) :
 865                                      outboundIpRewriteCache.contains(cacheKey);
 866
 867         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 868             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 869                          " is already done",
 870                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 871                          matchAddress, rewriteAddress, actionForNode);
 872             return;
 873         }
 874         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 875             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 876                          " is already done",
 877                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 878                          matchAddress, rewriteAddress, actionForNode);
 879             return;
 880         }
 881
 882         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 883                                                     matchAddress, rewriteAddress, actionForNode);
 884         if (status.isSuccess()) {
 885             // Update cache
 886             if (actionForNode == Action.ADD) {
 887                 if (isInbound) {
 888                     inboundIpRewriteCache.add(cacheKey);
 889                 } else {
 890                     outboundIpRewriteCache.add(cacheKey);
 891                 }
 892             } else {
 893                 if (isInbound) {
 894                     inboundIpRewriteCache.remove(cacheKey);
 895                 } else {
 896                     outboundIpRewriteCache.remove(cacheKey);
 897                 }
 898             }
 899         }
 900     }
 901
 902     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 903                                           final boolean isInbound,
 904                                           String matchAddress, String rewriteAddress,
 905                                           Action actionForNode) {
 906         Status status;
 907         try {
 908             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 909             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 910             if (isInbound) {
 911                 status = inboundNatProvider == null ?
 912                          new Status(StatusCode.SUCCESS) :
 913                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 914                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 915             } else {
 916                 status = outboundNatProvider == null ?
 917                          new Status(StatusCode.SUCCESS) :
 918                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 919                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 920             }
 921         } catch (UnknownHostException e) {
 922             status = new Status(StatusCode.BADREQUEST);
 923         }
 924
 925         if (status.isSuccess()) {
 926             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 927             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 928                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 929                          matchAddress, rewriteAddress, node, actionForNode);
 930         } else {
 931             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 932                          (isInbound ? "inbound" : "outbound"),
 933                          matchAddress, rewriteAddress, node, actionForNode, status);
 934         }
 935         return status;
 936     }
 937
 938     //
 939     // More Internals
 940     //
 941
 942     private int getMaskLenFromCidr(String cidr) {
 943         if (cidr == null) return 0;
 944         String[] splits = cidr.split("/");
 945         if (splits.length != 2) return 0;
 946
 947         int result;
 948         try {
 949             result = Integer.parseInt(splits[1].trim());
 950         }
 951         catch (NumberFormatException nfe)
 952         {
 953             result = 0;
 954         }
 955         return result;
 956     }
 957
 958     private Long getDpid (Node node) {
 959         Preconditions.checkNotNull(ovsdbConfigurationService);
 960
 961         String bridgeName = configurationService.getIntegrationBridgeName();
 962         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 963         if (bridgeUuid == null) {
 964             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 965             return 0L;
 966         }
 967
 968         try {
 969             Row bridgeRow =  ovsdbConfigurationService
 970                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 971             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 972             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 973             if (dpids == null || dpids.size() == 0) return 0L;
 974             return StringConvertor.dpidStringToLong((String) dpids.toArray()[0]);
 975         } catch (Exception e) {
 976             logger.error("Error finding Bridge's OF DPID", e);
 977             return 0L;
 978         }
 979     }
 980
 981     private String getInternalBridgeUUID (Node node, String bridgeName) {
 982         Preconditions.checkNotNull(ovsdbConfigurationService);
 983         try {
 984             Map<String, Row> bridgeTable =
 985                     ovsdbConfigurationService.getRows(node,
 986                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 987             if (bridgeTable == null) return null;
 988             for (String key : bridgeTable.keySet()) {
 989                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 990                 if (bridge.getName().equals(bridgeName)) return key;
 991             }
 992         } catch (Exception e) {
 993             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 994         }
 995         return null;
 996     }
 997 }