openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  13 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  14 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  15 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  16 import org.opendaylight.neutron.spi.NeutronNetwork;
  17 import org.opendaylight.neutron.spi.NeutronPort;
  18 import org.opendaylight.neutron.spi.NeutronRouter;
  19 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  20 import org.opendaylight.neutron.spi.NeutronSubnet;
  21 import org.opendaylight.neutron.spi.Neutron_IPs;
  22 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  24 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  26 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  27
  28 import com.google.common.base.Preconditions;
  29 import org.osgi.framework.BundleContext;
  30 import org.osgi.framework.ServiceReference;
  31 import org.slf4j.Logger;
  32 import org.slf4j.LoggerFactory;
  33
  34 import java.net.InetAddress;
  35 import java.net.UnknownHostException;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41
  42 /**
  43  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  44  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  45  * as well as the multi-tenant router forwarding provider.
  46  */
  47 public class NeutronL3Adapter implements ConfigInterface {
  48     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  49
  50     // The implementation for each of these services is resolved by the OSGi Service Manager
  51     private volatile ConfigurationService configurationService;
  52     private volatile TenantNetworkManager tenantNetworkManager;
  53     private volatile NodeCacheManager nodeCacheManager;
  54     private volatile INeutronNetworkCRUD neutronNetworkCache;
  55     private volatile INeutronSubnetCRUD neutronSubnetCache;
  56     private volatile INeutronPortCRUD neutronPortCache;
  57     private volatile L3ForwardingProvider l3ForwardingProvider;
  58     private volatile InboundNatProvider inboundNatProvider;
  59     private volatile OutboundNatProvider outboundNatProvider;
  60     private volatile ArpProvider arpProvider;
  61     private volatile RoutingProvider routingProvider;
  62
  63     private Set<String> inboundIpRewriteCache;
  64     private Set<String> outboundIpRewriteCache;
  65     private Set<String> inboundIpRewriteExclusionCache;
  66     private Set<String> outboundIpRewriteExclusionCache;
  67     private Set<String> routerInterfacesCache;
  68     private Set<String> staticArpEntryCache;
  69     private Set<String> l3ForwardingCache;
  70     private Set<String> defaultRouteCache;
  71     private Map<String, String> networkIdToRouterMacCache;
  72     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  73     private Boolean enabled = false;
  74     private Boolean flgDistributedARPEnabled = true;
  75     private Southbound southbound;
  76
  77     public NeutronL3Adapter() {
  78         logger.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
  79     }
  80
  81     private void initL3AdapterMembers() {
  82         Preconditions.checkNotNull(configurationService);
  83
  84         if (configurationService.isL3ForwardingEnabled()) {
  85             this.inboundIpRewriteCache = new HashSet<>();
  86             this.outboundIpRewriteCache = new HashSet<>();
  87             this.inboundIpRewriteExclusionCache = new HashSet<>();
  88             this.outboundIpRewriteExclusionCache = new HashSet<>();
  89             this.routerInterfacesCache = new HashSet<>();
  90             this.staticArpEntryCache = new HashSet<>();
  91             this.l3ForwardingCache = new HashSet<>();
  92             this.defaultRouteCache = new HashSet<>();
  93             this.networkIdToRouterMacCache = new HashMap<>();
  94             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  95             this.enabled = true;
  96             logger.info("OVSDB L3 forwarding is enabled");
  97             if (configurationService.isDistributedArpDisabled()) {
  98                 this.flgDistributedARPEnabled = false;
  99                 logger.info("Distributed ARP responder is disabled");
 100             } else {
 101                 logger.debug("Distributed ARP responder is enabled");
 102             }
 103         } else {
 104             logger.debug("OVSDB L3 forwarding is disabled");
 105         }
 106     }
 107
 108     //
 109     // Callbacks from OVSDB's northbound handlers
 110     //
 111
 112     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 113         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 114         if (!this.enabled)
 115             return;
 116     }
 117
 118     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 119         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 120         if (!this.enabled)
 121             return;
 122
 123         final boolean isDelete = action == Action.DELETE;
 124
 125         // Treat the port event as a router interface event if the port belongs to router. This is a
 126         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 127         //
 128         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 129             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 130                 NeutronRouter_Interface neutronRouterInterface =
 131                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 132                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 133                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 134
 135                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 136             }
 137         } else {
 138             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 139             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 140             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 141             // there.
 142             //
 143             if (!isDelete) {
 144                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 145                     NeutronRouter_Interface neutronRouterInterface =
 146                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 147                     if (neutronRouterInterface != null) {
 148                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 149                     }
 150                 }
 151             }
 152             this.updateL3ForNeutronPort(neutronPort, isDelete);
 153         }
 154     }
 155
 156     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 157         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 158         if (!this.enabled)
 159             return;
 160     }
 161
 162     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 163                                                   final NeutronRouter_Interface neutronRouterInterface,
 164                                                   Action action) {
 165         logger.debug("Router interface {} got event {}. Subnet {}",
 166                      neutronRouterInterface.getPortUUID(),
 167                      action,
 168                      neutronRouterInterface.getSubnetUUID());
 169         if (!this.enabled)
 170             return;
 171
 172         final boolean isDelete = action == Action.DELETE;
 173
 174         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 175
 176         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 177         // see if they are affected by l3
 178         //
 179         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 180             boolean currPortShouldBeDeleted = false;
 181             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 182             if (isDelete) {
 183                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 184                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 185                         currPortShouldBeDeleted = true;
 186                         break;
 187                     }
 188                 }
 189             }
 190             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 191         }
 192     }
 193
 194     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 195                                              Action action) {
 196         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 197                      neutronFloatingIP.getFixedIPAddress(),
 198                      neutronFloatingIP.getFloatingIPAddress(),
 199                      neutronFloatingIP.getFloatingNetworkUUID());
 200         if (!this.enabled)
 201             return;
 202
 203         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 204     }
 205
 206     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 207         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 208         if (!this.enabled)
 209             return;
 210     }
 211
 212     //
 213     // Callbacks from OVSDB's southbound handler
 214     //
 215     public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
 216                                      final NeutronNetwork neutronNetwork, Action action) {
 217         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 218                      action, node, intf.getName(), neutronNetwork);
 219         if (!this.enabled)
 220             return;
 221
 222         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 223         if (neutronPort != null) {
 224             this.handleNeutronPortEvent(neutronPort, action);
 225         }
 226     }
 227
 228     //
 229     // Internal helpers
 230     //
 231     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 232
 233         final String networkUUID = neutronPort.getNetworkUUID();
 234         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 235
 236         // If there is no router interface handling the networkUUID, we are done
 237         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 238             return;
 239         }
 240
 241         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 242         // router interface are handled via handleNeutronRouterInterfaceEvent.
 243         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 244             return;
 245         }
 246
 247         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 248         final String providerSegmentationId = neutronNetwork != null ?
 249                                               neutronNetwork.getProviderSegmentationID() : null;
 250         final String tenantMac = neutronPort.getMacAddress();
 251
 252         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 253             tenantMac == null || tenantMac.isEmpty()) {
 254             return;  // done: go no further w/out all the info needed...
 255         }
 256
 257         final Action action = isDelete ? Action.DELETE : Action.ADD;
 258         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 259         if (nodes.isEmpty()) {
 260             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 261         }
 262         for (Node node : nodes) {
 263             final Long dpid = getDpidForIntegrationBridge(node);
 264             if (dpid == null) {
 265                 continue;
 266             }
 267
 268             final boolean tenantNetworkPresentInNode =
 269                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 270             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 271                 final String tenantIpStr = neutronIP.getIpAddress();
 272                 if (tenantIpStr.isEmpty()) {
 273                     continue;
 274                 }
 275
 276                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 277                 // still needed when routing to subnets non-local to node (bug 2076).
 278                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 279
 280                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 281                 if (true == flgDistributedARPEnabled) {
 282                     programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 283                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 284                 }
 285             }
 286         }
 287     }
 288
 289     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 290                                            String macAddress, String ipStr,
 291                                            Action actionForNode) {
 292         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 293         // will look at desired action for node.
 294         //
 295         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 296         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 297
 298         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 299             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 300                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 301             return;
 302         }
 303         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 304             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 305                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 306             return;
 307         }
 308
 309         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 310                                                        macAddress, ipStr, actionForNode);
 311         if (status.isSuccess()) {
 312             // Update cache
 313             if (actionForNode == Action.ADD) {
 314                 l3ForwardingCache.add(cacheKey);
 315             } else {
 316                 l3ForwardingCache.remove(cacheKey);
 317             }
 318         }
 319     }
 320
 321     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 322                                              String macAddress,
 323                                              String address,
 324                                              Action actionForNode) {
 325         Status status;
 326         try {
 327             InetAddress inetAddress = InetAddress.getByName(address);
 328             status = l3ForwardingProvider == null ?
 329                      new Status(StatusCode.SUCCESS) :
 330                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 331                                                                       inetAddress, macAddress, actionForNode);
 332         } catch (UnknownHostException e) {
 333             status = new Status(StatusCode.BADREQUEST);
 334         }
 335
 336         if (status.isSuccess()) {
 337             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 338                          l3ForwardingProvider == null ? "skipped" : "programmed",
 339                          macAddress, address, node, actionForNode);
 340         } else {
 341             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 342                          macAddress, address, node, actionForNode, status);
 343         }
 344         return status;
 345     }
 346
 347     // --
 348
 349     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 350                                                        Boolean isDelete) {
 351         Preconditions.checkNotNull(destNeutronRouterInterface);
 352
 353         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 354         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 355         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 356         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 357         final NeutronNetwork neutronNetwork = subnet != null ?
 358                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 359         final String destinationSegmentationId = neutronNetwork != null ?
 360                                                  neutronNetwork.getProviderSegmentationID() : null;
 361         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 362         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 363         final String cidr = subnet != null ? subnet.getCidr() : null;
 364         final int mask = getMaskLenFromCidr(cidr);
 365
 366         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 367                      destNeutronRouterInterface, isDelete);
 368
 369         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 370             cidr == null || cidr.isEmpty() ||
 371             macAddress == null || macAddress.isEmpty() ||
 372             ipList == null || ipList.isEmpty()) {
 373             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 374                          destinationSegmentationId, cidr, macAddress, ipList);
 375             return;  // done: go no further w/out all the info needed...
 376         }
 377
 378         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 379
 380         // Keep cache for finding router's mac from network uuid -- add
 381         //
 382         if (! isDelete) {
 383             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 384             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 385         }
 386
 387         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 388         if (nodes.isEmpty()) {
 389             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 390         }
 391         for (Node node : nodes) {
 392             final Long dpid = getDpidForIntegrationBridge(node);
 393             if (dpid == null) {
 394                 continue;
 395             }
 396
 397             for (Neutron_IPs neutronIP : ipList) {
 398                 final String ipStr = neutronIP.getIpAddress();
 399                 if (ipStr.isEmpty()) {
 400                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 401                             node.getNodeId().getValue(), ipStr);
 402                     continue;
 403                 }
 404
 405                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 406                 //
 407                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 408                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 409                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 410                                                               neutronNetwork, destinationSegmentationId,
 411                                                               macAddress, ipStr, mask, actionForNode,
 412                                                               true /*isReflexsive*/);
 413                 }
 414
 415                     // Enable ARP responder by default, because router interface needs to be responded always.
 416                     programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 417             }
 418
 419             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 420             // for the external neutron networks.
 421             //
 422             {
 423                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 424                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 425                                                 cidr, actionForRewriteExclusion);
 426                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 427                                                 cidr, actionForRewriteExclusion);
 428             }
 429
 430             // Default route. For non-external subnet, make sure that there is none configured.
 431             //
 432             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 433                 final Action actionForNodeDefaultRoute =
 434                         isExternal ? actionForNode : Action.DELETE;
 435                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 436                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 437                                           actionForNodeDefaultRoute);
 438             }
 439         }
 440
 441         // Keep cache for finding router's mac from network uuid -- remove
 442         //
 443         if (isDelete) {
 444             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 445             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 446         }
 447     }
 448
 449     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 450                                                            final Long dpid,
 451                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 452                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 453                                                            final NeutronNetwork dstNeutronNetwork,
 454                                                            final String destinationSegmentationId,
 455                                                            final String dstMacAddress,
 456                                                            final String destIpStr,
 457                                                            final int destMask,
 458                                                            final Action actionForNode,
 459                                                            Boolean isReflexsive) {
 460         Preconditions.checkNotNull(srcNeutronRouterInterface);
 461         Preconditions.checkNotNull(dstNeutronRouterInterface);
 462
 463         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 464         if (sourceSubnetId == null) {
 465             logger.error("Could not get provider Subnet ID from router interface {}",
 466                          srcNeutronRouterInterface.getID());
 467             return;
 468         }
 469
 470         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 471         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 472         if (sourceNetworkId == null) {
 473             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 474             return;
 475         }
 476
 477         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 478         if (sourceNetwork == null) {
 479             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 480             return;
 481         }
 482
 483         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 484             // Isolate subnets from different tenants within the same router
 485             return;
 486         }
 487         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 488         if (sourceSegmentationId == null) {
 489             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 490             return;
 491         }
 492         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 493             // Skip 'self'
 494             return;
 495         }
 496
 497         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 498                                      dstMacAddress, destIpStr, destMask, actionForNode);
 499
 500         // Flip roles src->dst; dst->src
 501         if (isReflexsive) {
 502             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 503             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 504             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 505             final String cidr2 = sourceSubnet.getCidr();
 506             final int mask2 = getMaskLenFromCidr(cidr2);
 507
 508             if (cidr2 == null || cidr2.isEmpty() ||
 509                 macAddress2 == null || macAddress2.isEmpty() ||
 510                 ipList2 == null || ipList2.isEmpty()) {
 511                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 512                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 513                 return;  // done: go no further w/out all the info needed...
 514             }
 515
 516             for (Neutron_IPs neutronIP2 : ipList2) {
 517                 final String ipStr2 = neutronIP2.getIpAddress();
 518                 if (ipStr2.isEmpty()) {
 519                     continue;
 520                 }
 521                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 522                                                           srcNeutronRouterInterface,
 523                                                           sourceNetwork, sourceSegmentationId,
 524                                                           macAddress2, ipStr2, mask2, actionForNode,
 525                                                           false /*isReflexsive*/);
 526             }
 527         }
 528     }
 529
 530     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 531                                               String destinationSegmentationId,
 532                                               String macAddress, String ipStr, int mask,
 533                                               Action actionForNode) {
 534         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 535         // will look at desired action for node.
 536         //
 537         final String cacheKey = node.getNodeId().getValue() + ":" +
 538                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 539                                 ipStr + "/" + Integer.toString(mask);
 540         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 541
 542         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 543             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 544                          " action {} is already done",
 545                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 546                          macAddress, ipStr, mask, actionForNode);
 547             return;
 548         }
 549         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 550             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 551                          " action {} is already done",
 552                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 553                          macAddress, ipStr, mask, actionForNode);
 554             return;
 555         }
 556
 557         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 558                                                           macAddress, ipStr, mask, actionForNode);
 559         if (status.isSuccess()) {
 560             // Update cache
 561             if (actionForNode == Action.ADD) {
 562                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 563                 routerInterfacesCache.add(cacheKey);
 564             } else {
 565                 // TODO: multiTenantAwareRouter.removeInterface(...);
 566                 routerInterfacesCache.remove(cacheKey);
 567             }
 568         }
 569     }
 570
 571     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 572                                                 String destinationSegmentationId,
 573                                                 String macAddress,
 574                                                 String address, int mask,
 575                                                 Action actionForNode) {
 576         Status status;
 577         try {
 578             InetAddress inetAddress = InetAddress.getByName(address);
 579             status = routingProvider == null ?
 580                      new Status(StatusCode.SUCCESS) :
 581                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 582                                                             macAddress, inetAddress, mask, actionForNode);
 583         } catch (UnknownHostException e) {
 584             status = new Status(StatusCode.BADREQUEST);
 585         }
 586
 587         if (status.isSuccess()) {
 588             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 589                          routingProvider == null ? "skipped" : "programmed",
 590                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 591                          actionForNode);
 592         } else {
 593             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 594                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 595                          actionForNode, status);
 596         }
 597         return status;
 598     }
 599
 600     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 601                                         String macAddress, String ipStr,
 602                                         Action actionForNode) {
 603         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 604         // will look at desired action for node.
 605         //
 606         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 607         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 608
 609         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 610             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 611                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 612             return;
 613         }
 614         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 615             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 616                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 617             return;
 618         }
 619
 620         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 621                                                     macAddress, ipStr, actionForNode);
 622         if (status.isSuccess()) {
 623             // Update cache
 624             if (actionForNode == Action.ADD) {
 625                 staticArpEntryCache.add(cacheKey);
 626             } else {
 627                 staticArpEntryCache.remove(cacheKey);
 628             }
 629         }
 630     }
 631
 632     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 633                                                 String macAddress,
 634                                                 String address,
 635                                                 Action actionForNode) {
 636         Status status;
 637         try {
 638             InetAddress inetAddress = InetAddress.getByName(address);
 639             status = arpProvider == null ?
 640                      new Status(StatusCode.SUCCESS) :
 641                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 642                                                        macAddress, inetAddress, actionForNode);
 643         } catch (UnknownHostException e) {
 644             status = new Status(StatusCode.BADREQUEST);
 645         }
 646
 647         if (status.isSuccess()) {
 648             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 649                          arpProvider == null ? "skipped" : "programmed",
 650                          macAddress, address, node, actionForNode);
 651         } else {
 652             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 653                          macAddress, address, node, actionForNode, status);
 654         }
 655         return status;
 656     }
 657
 658     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 659                                                  final boolean isInbound, String cidr,
 660                                                  Action actionForRewriteExclusion) {
 661         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 662         // will look at desired action for node.
 663         //
 664         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
 665         final Boolean isProgrammed = isInbound ?
 666                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 667                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 668
 669         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 670             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 671                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 672                          actionForRewriteExclusion);
 673             return;
 674         }
 675         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 676             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 677                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 678                          actionForRewriteExclusion);
 679             return;
 680         }
 681
 682         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 683                                                              isInbound, actionForRewriteExclusion);
 684         if (status.isSuccess()) {
 685             // Update cache
 686             if (actionForRewriteExclusion == Action.ADD) {
 687                 if (isInbound) {
 688                     inboundIpRewriteExclusionCache.add(cacheKey);
 689                 } else {
 690                     outboundIpRewriteExclusionCache.add(cacheKey);
 691                 }
 692             } else {
 693                 if (isInbound) {
 694                     inboundIpRewriteExclusionCache.remove(cacheKey);
 695                 } else {
 696                     outboundIpRewriteExclusionCache.remove(cacheKey);
 697                 }
 698             }
 699         }
 700     }
 701
 702     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 703                                                    final boolean isInbound, Action actionForNode) {
 704         Status status;
 705         if (isInbound) {
 706             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 707                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 708                                                                   actionForNode);
 709         } else {
 710             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 711                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 712                                                                    actionForNode);
 713         }
 714
 715         if (status.isSuccess()) {
 716             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 717             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 718                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 719                          cidr, node, actionForNode);
 720         } else {
 721             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 722                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 723         }
 724         return status;
 725     }
 726
 727     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 728                                            String defaultGatewayMacAddress, String gatewayIp,
 729                                            Action actionForNodeDefaultRoute) {
 730         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 731         // will look at desired action for node.
 732         //
 733         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + gatewayIp;
 734         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 735
 736         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 737             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 738                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 739                          actionForNodeDefaultRoute);
 740             return;
 741         }
 742         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 743             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 744                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 745                          actionForNodeDefaultRoute);
 746             return;
 747         }
 748
 749         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 750                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 751         if (status.isSuccess()) {
 752             // Update cache
 753             if (actionForNodeDefaultRoute == Action.ADD) {
 754                 defaultRouteCache.add(cacheKey);
 755             } else {
 756                 defaultRouteCache.remove(cacheKey);
 757             }
 758         }
 759     }
 760
 761     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 762                                           String defaultGatewayMacAddress,
 763                                           String gatewayIp,
 764                                           Action actionForNodeDefaultRoute) {
 765         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 766         if (defaultGatewayMacAddress == null) {
 767             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 768                          gatewayIp, node, actionForNodeDefaultRoute);
 769             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 770         }
 771
 772         Status status;
 773         try {
 774             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 775             status = routingProvider == null ?
 776                      new Status(StatusCode.SUCCESS) :
 777                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 778                                                               defaultGatewayMacAddress, inetAddress,
 779                                                               actionForNodeDefaultRoute);
 780         } catch (UnknownHostException e) {
 781             status = new Status(StatusCode.BADREQUEST);
 782         }
 783
 784         if (status.isSuccess()) {
 785             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 786                          routingProvider == null ? "skipped" : "programmed",
 787                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 788         } else {
 789             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 790                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 791         }
 792         return status;
 793     }
 794
 795     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 796         Preconditions.checkNotNull(neutronFloatingIP);
 797
 798         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 799         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 800
 801         // If there is no router interface handling the networkUUID, we are done
 802         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 803             return;
 804         }
 805
 806         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 807         final String providerSegmentationId = neutronNetwork != null ?
 808                                               neutronNetwork.getProviderSegmentationID() : null;
 809         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 810         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 811
 812         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 813             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 814             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 815             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 816             return;  // done: go no further w/out all the info needed...
 817         }
 818
 819         final Action action = isDelete ? Action.DELETE : Action.ADD;
 820         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 821         if (nodes.isEmpty()) {
 822             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 823         }
 824         for (Node node : nodes) {
 825             final Long dpid = getDpidForIntegrationBridge(node);
 826             if (dpid == null) {
 827                 continue;
 828             }
 829
 830             final Action actionForNode =
 831                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 832                     action : Action.DELETE;
 833
 834             // Rewrite from float to fixed and vice-versa
 835             //
 836             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 837                                    floatingIpAddress, fixedIPAddress, actionForNode);
 838             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 839                                    fixedIPAddress, floatingIpAddress, actionForNode);
 840
 841             // Respond to ARPs for the floating ip address by default
 842             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 843                                    actionForNode);
 844         }
 845     }
 846
 847     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 848                                         final boolean isInbound,
 849                                         String matchAddress, String rewriteAddress,
 850                                         Action actionForNode) {
 851         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 852         // will look at desired action for node.
 853         //
 854         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" +
 855                                 matchAddress + ":" + rewriteAddress;
 856         final Boolean isProgrammed = isInbound ?
 857                                      inboundIpRewriteCache.contains(cacheKey) :
 858                                      outboundIpRewriteCache.contains(cacheKey);
 859
 860         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 861             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 862                          " is already done",
 863                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 864                          matchAddress, rewriteAddress, actionForNode);
 865             return;
 866         }
 867         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 868             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 869                          " is already done",
 870                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 871                          matchAddress, rewriteAddress, actionForNode);
 872             return;
 873         }
 874
 875         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 876                                                     matchAddress, rewriteAddress, actionForNode);
 877         if (status.isSuccess()) {
 878             // Update cache
 879             if (actionForNode == Action.ADD) {
 880                 if (isInbound) {
 881                     inboundIpRewriteCache.add(cacheKey);
 882                 } else {
 883                     outboundIpRewriteCache.add(cacheKey);
 884                 }
 885             } else {
 886                 if (isInbound) {
 887                     inboundIpRewriteCache.remove(cacheKey);
 888                 } else {
 889                     outboundIpRewriteCache.remove(cacheKey);
 890                 }
 891             }
 892         }
 893     }
 894
 895     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 896                                           final boolean isInbound,
 897                                           String matchAddress, String rewriteAddress,
 898                                           Action actionForNode) {
 899         Status status;
 900         try {
 901             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 902             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 903             if (isInbound) {
 904                 status = inboundNatProvider == null ?
 905                          new Status(StatusCode.SUCCESS) :
 906                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 907                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 908             } else {
 909                 status = outboundNatProvider == null ?
 910                          new Status(StatusCode.SUCCESS) :
 911                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 912                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 913             }
 914         } catch (UnknownHostException e) {
 915             status = new Status(StatusCode.BADREQUEST);
 916         }
 917
 918         if (status.isSuccess()) {
 919             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 920             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 921                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 922                          matchAddress, rewriteAddress, node, actionForNode);
 923         } else {
 924             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 925                          (isInbound ? "inbound" : "outbound"),
 926                          matchAddress, rewriteAddress, node, actionForNode, status);
 927         }
 928         return status;
 929     }
 930
 931     private int getMaskLenFromCidr(String cidr) {
 932         if (cidr == null) return 0;
 933         String[] splits = cidr.split("/");
 934         if (splits.length != 2) return 0;
 935
 936         int result;
 937         try {
 938             result = Integer.parseInt(splits[1].trim());
 939         }
 940         catch (NumberFormatException nfe)
 941         {
 942             result = 0;
 943         }
 944         return result;
 945     }
 946
 947     private Long getDpidForIntegrationBridge(Node node) {
 948         // Check if node is integration bridge; and only then return its dpid
 949         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
 950             return southbound.getDataPathId(node);
 951         }
 952         return null;
 953     }
 954
 955     @Override
 956     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
 957         tenantNetworkManager =
 958                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
 959         configurationService =
 960                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
 961         arpProvider =
 962                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
 963         inboundNatProvider =
 964                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
 965         outboundNatProvider =
 966                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
 967         routingProvider =
 968                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
 969         l3ForwardingProvider =
 970                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
 971         nodeCacheManager =
 972                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
 973         southbound =
 974                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
 975
 976         initL3AdapterMembers();
 977     }
 978
 979     @Override
 980     public void setDependencies(Object impl) {
 981         if (impl instanceof INeutronNetworkCRUD) {
 982             neutronNetworkCache = (INeutronNetworkCRUD)impl;
 983         } else if (impl instanceof INeutronPortCRUD) {
 984             neutronPortCache = (INeutronPortCRUD)impl;
 985         } else if (impl instanceof INeutronSubnetCRUD) {
 986             neutronSubnetCache = (INeutronSubnetCRUD)impl;
 987         } else if (impl instanceof ArpProvider) {
 988             arpProvider = (ArpProvider)impl;
 989         } else if (impl instanceof InboundNatProvider) {
 990             inboundNatProvider = (InboundNatProvider)impl;
 991         } else if (impl instanceof OutboundNatProvider) {
 992             outboundNatProvider = (OutboundNatProvider)impl;
 993         } else if (impl instanceof RoutingProvider) {
 994             routingProvider = (RoutingProvider)impl;
 995         } else if (impl instanceof L3ForwardingProvider) {
 996             l3ForwardingProvider = (L3ForwardingProvider)impl;
 997         }
 998     }
 999 }