openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  13 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  14 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  15 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  16 import org.opendaylight.neutron.spi.NeutronNetwork;
  17 import org.opendaylight.neutron.spi.NeutronPort;
  18 import org.opendaylight.neutron.spi.NeutronRouter;
  19 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  20 import org.opendaylight.neutron.spi.NeutronSubnet;
  21 import org.opendaylight.neutron.spi.Neutron_IPs;
  22 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  24 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  25 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  26 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  27
  28 import com.google.common.base.Preconditions;
  29 import org.osgi.framework.BundleContext;
  30 import org.osgi.framework.ServiceReference;
  31 import org.slf4j.Logger;
  32 import org.slf4j.LoggerFactory;
  33
  34 import java.net.InetAddress;
  35 import java.net.UnknownHostException;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.List;
  39 import java.util.Map;
  40 import java.util.Set;
  41
  42 /**
  43  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  44  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  45  * as well as the multi-tenant router forwarding provider.
  46  */
  47 public class NeutronL3Adapter implements ConfigInterface {
  48     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  49
  50     // The implementation for each of these services is resolved by the OSGi Service Manager
  51     private volatile ConfigurationService configurationService;
  52     private volatile TenantNetworkManager tenantNetworkManager;
  53     private volatile NodeCacheManager nodeCacheManager;
  54     private volatile INeutronNetworkCRUD neutronNetworkCache;
  55     private volatile INeutronSubnetCRUD neutronSubnetCache;
  56     private volatile INeutronPortCRUD neutronPortCache;
  57     private volatile L3ForwardingProvider l3ForwardingProvider;
  58     private volatile InboundNatProvider inboundNatProvider;
  59     private volatile OutboundNatProvider outboundNatProvider;
  60     private volatile ArpProvider arpProvider;
  61     private volatile RoutingProvider routingProvider;
  62
  63     private Set<String> inboundIpRewriteCache;
  64     private Set<String> outboundIpRewriteCache;
  65     private Set<String> inboundIpRewriteExclusionCache;
  66     private Set<String> outboundIpRewriteExclusionCache;
  67     private Set<String> routerInterfacesCache;
  68     private Set<String> staticArpEntryCache;
  69     private Set<String> l3ForwardingCache;
  70     private Set<String> defaultRouteCache;
  71     private Map<String, String> networkIdToRouterMacCache;
  72     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  73     private Boolean enabled = false;
  74     private Southbound southbound;
  75
  76     public NeutronL3Adapter() {
  77         logger.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
  78     }
  79
  80     private void initL3AdapterMembers() {
  81         Preconditions.checkNotNull(configurationService);
  82
  83         if (configurationService.isL3ForwardingEnabled()) {
  84             this.inboundIpRewriteCache = new HashSet<>();
  85             this.outboundIpRewriteCache = new HashSet<>();
  86             this.inboundIpRewriteExclusionCache = new HashSet<>();
  87             this.outboundIpRewriteExclusionCache = new HashSet<>();
  88             this.routerInterfacesCache = new HashSet<>();
  89             this.staticArpEntryCache = new HashSet<>();
  90             this.l3ForwardingCache = new HashSet<>();
  91             this.defaultRouteCache = new HashSet<>();
  92             this.networkIdToRouterMacCache = new HashMap<>();
  93             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  94
  95             this.enabled = true;
  96             logger.info("OVSDB L3 forwarding is enabled");
  97         } else {
  98             logger.debug("OVSDB L3 forwarding is disabled");
  99         }
 100     }
 101
 102     //
 103     // Callbacks from OVSDB's northbound handlers
 104     //
 105
 106     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 107         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 108         if (!this.enabled)
 109             return;
 110     }
 111
 112     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 113         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 114         if (!this.enabled)
 115             return;
 116
 117         final boolean isDelete = action == Action.DELETE;
 118
 119         // Treat the port event as a router interface event if the port belongs to router. This is a
 120         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 121         //
 122         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 123             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 124                 NeutronRouter_Interface neutronRouterInterface =
 125                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 126                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 127                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 128
 129                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 130             }
 131         } else {
 132             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 133             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 134             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 135             // there.
 136             //
 137             if (!isDelete) {
 138                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 139                     NeutronRouter_Interface neutronRouterInterface =
 140                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 141                     if (neutronRouterInterface != null) {
 142                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 143                     }
 144                 }
 145             }
 146             this.updateL3ForNeutronPort(neutronPort, isDelete);
 147         }
 148     }
 149
 150     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 151         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 152         if (!this.enabled)
 153             return;
 154     }
 155
 156     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 157                                                   final NeutronRouter_Interface neutronRouterInterface,
 158                                                   Action action) {
 159         logger.debug("Router interface {} got event {}. Subnet {}",
 160                      neutronRouterInterface.getPortUUID(),
 161                      action,
 162                      neutronRouterInterface.getSubnetUUID());
 163         if (!this.enabled)
 164             return;
 165
 166         final boolean isDelete = action == Action.DELETE;
 167
 168         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 169
 170         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 171         // see if they are affected by l3
 172         //
 173         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 174             boolean currPortShouldBeDeleted = false;
 175             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 176             if (isDelete) {
 177                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 178                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 179                         currPortShouldBeDeleted = true;
 180                         break;
 181                     }
 182                 }
 183             }
 184             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 185         }
 186     }
 187
 188     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 189                                              Action action) {
 190         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 191                      neutronFloatingIP.getFixedIPAddress(),
 192                      neutronFloatingIP.getFloatingIPAddress(),
 193                      neutronFloatingIP.getFloatingNetworkUUID());
 194         if (!this.enabled)
 195             return;
 196
 197         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 198     }
 199
 200     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 201         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 202         if (!this.enabled)
 203             return;
 204     }
 205
 206     //
 207     // Callbacks from OVSDB's southbound handler
 208     //
 209     public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
 210                                      final NeutronNetwork neutronNetwork, Action action) {
 211         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 212                      action, node, intf.getName(), neutronNetwork);
 213         if (!this.enabled)
 214             return;
 215
 216         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 217         if (neutronPort != null) {
 218             this.handleNeutronPortEvent(neutronPort, action);
 219         }
 220     }
 221
 222     //
 223     // Internal helpers
 224     //
 225     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 226
 227         final String networkUUID = neutronPort.getNetworkUUID();
 228         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 229
 230         // If there is no router interface handling the networkUUID, we are done
 231         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 232             return;
 233         }
 234
 235         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 236         // router interface are handled via handleNeutronRouterInterfaceEvent.
 237         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 238             return;
 239         }
 240
 241         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 242         final String providerSegmentationId = neutronNetwork != null ?
 243                                               neutronNetwork.getProviderSegmentationID() : null;
 244         final String tenantMac = neutronPort.getMacAddress();
 245
 246         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 247             tenantMac == null || tenantMac.isEmpty()) {
 248             return;  // done: go no further w/out all the info needed...
 249         }
 250
 251         final Action action = isDelete ? Action.DELETE : Action.ADD;
 252         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 253         if (nodes.isEmpty()) {
 254             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 255         }
 256         for (Node node : nodes) {
 257             final Long dpid = getDpidForIntegrationBridge(node);
 258             if (dpid == null) {
 259                 continue;
 260             }
 261
 262             final boolean tenantNetworkPresentInNode =
 263                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 264             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 265                 final String tenantIpStr = neutronIP.getIpAddress();
 266                 if (tenantIpStr.isEmpty()) {
 267                     continue;
 268                 }
 269
 270                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 271                 // still needed when routing to subnets non-local to node (bug 2076).
 272                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 273
 274                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 275                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 276                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 277             }
 278         }
 279     }
 280
 281     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 282                                            String macAddress, String ipStr,
 283                                            Action actionForNode) {
 284         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 285         // will look at desired action for node.
 286         //
 287         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 288         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 289
 290         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 291             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 292                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 293             return;
 294         }
 295         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 296             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 297                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 298             return;
 299         }
 300
 301         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 302                                                        macAddress, ipStr, actionForNode);
 303         if (status.isSuccess()) {
 304             // Update cache
 305             if (actionForNode == Action.ADD) {
 306                 l3ForwardingCache.add(cacheKey);
 307             } else {
 308                 l3ForwardingCache.remove(cacheKey);
 309             }
 310         }
 311     }
 312
 313     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 314                                              String macAddress,
 315                                              String address,
 316                                              Action actionForNode) {
 317         Status status;
 318         try {
 319             InetAddress inetAddress = InetAddress.getByName(address);
 320             status = l3ForwardingProvider == null ?
 321                      new Status(StatusCode.SUCCESS) :
 322                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 323                                                                       inetAddress, macAddress, actionForNode);
 324         } catch (UnknownHostException e) {
 325             status = new Status(StatusCode.BADREQUEST);
 326         }
 327
 328         if (status.isSuccess()) {
 329             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 330                          l3ForwardingProvider == null ? "skipped" : "programmed",
 331                          macAddress, address, node, actionForNode);
 332         } else {
 333             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 334                          macAddress, address, node, actionForNode, status);
 335         }
 336         return status;
 337     }
 338
 339     // --
 340
 341     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 342                                                        Boolean isDelete) {
 343         Preconditions.checkNotNull(destNeutronRouterInterface);
 344
 345         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 346         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 347         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 348         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 349         final NeutronNetwork neutronNetwork = subnet != null ?
 350                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 351         final String destinationSegmentationId = neutronNetwork != null ?
 352                                                  neutronNetwork.getProviderSegmentationID() : null;
 353         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 354         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 355         final String cidr = subnet != null ? subnet.getCidr() : null;
 356         final int mask = getMaskLenFromCidr(cidr);
 357
 358         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 359                      destNeutronRouterInterface, isDelete);
 360
 361         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 362             cidr == null || cidr.isEmpty() ||
 363             macAddress == null || macAddress.isEmpty() ||
 364             ipList == null || ipList.isEmpty()) {
 365             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 366                          destinationSegmentationId, cidr, macAddress, ipList);
 367             return;  // done: go no further w/out all the info needed...
 368         }
 369
 370         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 371
 372         // Keep cache for finding router's mac from network uuid -- add
 373         //
 374         if (! isDelete) {
 375             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 376             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 377         }
 378
 379         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 380         if (nodes.isEmpty()) {
 381             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 382         }
 383         for (Node node : nodes) {
 384             final Long dpid = getDpidForIntegrationBridge(node);
 385             if (dpid == null) {
 386                 continue;
 387             }
 388
 389             for (Neutron_IPs neutronIP : ipList) {
 390                 final String ipStr = neutronIP.getIpAddress();
 391                 if (ipStr.isEmpty()) {
 392                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 393                             node.getNodeId().getValue(), ipStr);
 394                     continue;
 395                 }
 396
 397                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 398                 //
 399                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 400                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 401                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 402                                                               neutronNetwork, destinationSegmentationId,
 403                                                               macAddress, ipStr, mask, actionForNode,
 404                                                               true /*isReflexsive*/);
 405                 }
 406
 407                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 408             }
 409
 410             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 411             // for the external neutron networks.
 412             //
 413             {
 414                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 415                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 416                                                 cidr, actionForRewriteExclusion);
 417                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 418                                                 cidr, actionForRewriteExclusion);
 419             }
 420
 421             // Default route. For non-external subnet, make sure that there is none configured.
 422             //
 423             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 424                 final Action actionForNodeDefaultRoute =
 425                         isExternal ? actionForNode : Action.DELETE;
 426                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 427                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 428                                           actionForNodeDefaultRoute);
 429             }
 430         }
 431
 432         // Keep cache for finding router's mac from network uuid -- remove
 433         //
 434         if (isDelete) {
 435             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 436             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 437         }
 438     }
 439
 440     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 441                                                            final Long dpid,
 442                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 443                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 444                                                            final NeutronNetwork dstNeutronNetwork,
 445                                                            final String destinationSegmentationId,
 446                                                            final String dstMacAddress,
 447                                                            final String destIpStr,
 448                                                            final int destMask,
 449                                                            final Action actionForNode,
 450                                                            Boolean isReflexsive) {
 451         Preconditions.checkNotNull(srcNeutronRouterInterface);
 452         Preconditions.checkNotNull(dstNeutronRouterInterface);
 453
 454         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 455         if (sourceSubnetId == null) {
 456             logger.error("Could not get provider Subnet ID from router interface {}",
 457                          srcNeutronRouterInterface.getID());
 458             return;
 459         }
 460
 461         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 462         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 463         if (sourceNetworkId == null) {
 464             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 465             return;
 466         }
 467
 468         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 469         if (sourceNetwork == null) {
 470             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 471             return;
 472         }
 473
 474         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 475             // Isolate subnets from different tenants within the same router
 476             return;
 477         }
 478         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 479         if (sourceSegmentationId == null) {
 480             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 481             return;
 482         }
 483         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 484             // Skip 'self'
 485             return;
 486         }
 487
 488         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 489                                      dstMacAddress, destIpStr, destMask, actionForNode);
 490
 491         // Flip roles src->dst; dst->src
 492         if (isReflexsive) {
 493             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 494             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 495             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 496             final String cidr2 = sourceSubnet.getCidr();
 497             final int mask2 = getMaskLenFromCidr(cidr2);
 498
 499             if (cidr2 == null || cidr2.isEmpty() ||
 500                 macAddress2 == null || macAddress2.isEmpty() ||
 501                 ipList2 == null || ipList2.isEmpty()) {
 502                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 503                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 504                 return;  // done: go no further w/out all the info needed...
 505             }
 506
 507             for (Neutron_IPs neutronIP2 : ipList2) {
 508                 final String ipStr2 = neutronIP2.getIpAddress();
 509                 if (ipStr2.isEmpty()) {
 510                     continue;
 511                 }
 512                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 513                                                           srcNeutronRouterInterface,
 514                                                           sourceNetwork, sourceSegmentationId,
 515                                                           macAddress2, ipStr2, mask2, actionForNode,
 516                                                           false /*isReflexsive*/);
 517             }
 518         }
 519     }
 520
 521     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 522                                               String destinationSegmentationId,
 523                                               String macAddress, String ipStr, int mask,
 524                                               Action actionForNode) {
 525         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 526         // will look at desired action for node.
 527         //
 528         final String cacheKey = node.getNodeId().getValue() + ":" +
 529                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 530                                 ipStr + "/" + Integer.toString(mask);
 531         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 532
 533         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 534             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 535                          " action {} is already done",
 536                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 537                          macAddress, ipStr, mask, actionForNode);
 538             return;
 539         }
 540         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 541             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 542                          " action {} is already done",
 543                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 544                          macAddress, ipStr, mask, actionForNode);
 545             return;
 546         }
 547
 548         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 549                                                           macAddress, ipStr, mask, actionForNode);
 550         if (status.isSuccess()) {
 551             // Update cache
 552             if (actionForNode == Action.ADD) {
 553                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 554                 routerInterfacesCache.add(cacheKey);
 555             } else {
 556                 // TODO: multiTenantAwareRouter.removeInterface(...);
 557                 routerInterfacesCache.remove(cacheKey);
 558             }
 559         }
 560     }
 561
 562     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 563                                                 String destinationSegmentationId,
 564                                                 String macAddress,
 565                                                 String address, int mask,
 566                                                 Action actionForNode) {
 567         Status status;
 568         try {
 569             InetAddress inetAddress = InetAddress.getByName(address);
 570             status = routingProvider == null ?
 571                      new Status(StatusCode.SUCCESS) :
 572                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 573                                                             macAddress, inetAddress, mask, actionForNode);
 574         } catch (UnknownHostException e) {
 575             status = new Status(StatusCode.BADREQUEST);
 576         }
 577
 578         if (status.isSuccess()) {
 579             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 580                          routingProvider == null ? "skipped" : "programmed",
 581                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 582                          actionForNode);
 583         } else {
 584             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 585                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 586                          actionForNode, status);
 587         }
 588         return status;
 589     }
 590
 591     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 592                                         String macAddress, String ipStr,
 593                                         Action actionForNode) {
 594         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 595         // will look at desired action for node.
 596         //
 597         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 598         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 599
 600         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 601             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 602                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 603             return;
 604         }
 605         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 606             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 607                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 608             return;
 609         }
 610
 611         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 612                                                     macAddress, ipStr, actionForNode);
 613         if (status.isSuccess()) {
 614             // Update cache
 615             if (actionForNode == Action.ADD) {
 616                 staticArpEntryCache.add(cacheKey);
 617             } else {
 618                 staticArpEntryCache.remove(cacheKey);
 619             }
 620         }
 621     }
 622
 623     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 624                                                 String macAddress,
 625                                                 String address,
 626                                                 Action actionForNode) {
 627         Status status;
 628         try {
 629             InetAddress inetAddress = InetAddress.getByName(address);
 630             status = arpProvider == null ?
 631                      new Status(StatusCode.SUCCESS) :
 632                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 633                                                        macAddress, inetAddress, actionForNode);
 634         } catch (UnknownHostException e) {
 635             status = new Status(StatusCode.BADREQUEST);
 636         }
 637
 638         if (status.isSuccess()) {
 639             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 640                          arpProvider == null ? "skipped" : "programmed",
 641                          macAddress, address, node, actionForNode);
 642         } else {
 643             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 644                          macAddress, address, node, actionForNode, status);
 645         }
 646         return status;
 647     }
 648
 649     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 650                                                  final boolean isInbound, String cidr,
 651                                                  Action actionForRewriteExclusion) {
 652         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 653         // will look at desired action for node.
 654         //
 655         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
 656         final Boolean isProgrammed = isInbound ?
 657                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 658                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 659
 660         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 661             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 662                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 663                          actionForRewriteExclusion);
 664             return;
 665         }
 666         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 667             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 668                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 669                          actionForRewriteExclusion);
 670             return;
 671         }
 672
 673         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 674                                                              isInbound, actionForRewriteExclusion);
 675         if (status.isSuccess()) {
 676             // Update cache
 677             if (actionForRewriteExclusion == Action.ADD) {
 678                 if (isInbound) {
 679                     inboundIpRewriteExclusionCache.add(cacheKey);
 680                 } else {
 681                     outboundIpRewriteExclusionCache.add(cacheKey);
 682                 }
 683             } else {
 684                 if (isInbound) {
 685                     inboundIpRewriteExclusionCache.remove(cacheKey);
 686                 } else {
 687                     outboundIpRewriteExclusionCache.remove(cacheKey);
 688                 }
 689             }
 690         }
 691     }
 692
 693     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 694                                                    final boolean isInbound, Action actionForNode) {
 695         Status status;
 696         if (isInbound) {
 697             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 698                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 699                                                                   actionForNode);
 700         } else {
 701             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 702                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 703                                                                    actionForNode);
 704         }
 705
 706         if (status.isSuccess()) {
 707             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 708             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 709                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 710                          cidr, node, actionForNode);
 711         } else {
 712             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 713                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 714         }
 715         return status;
 716     }
 717
 718     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 719                                            String defaultGatewayMacAddress, String gatewayIp,
 720                                            Action actionForNodeDefaultRoute) {
 721         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 722         // will look at desired action for node.
 723         //
 724         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + gatewayIp;
 725         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 726
 727         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 728             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 729                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 730                          actionForNodeDefaultRoute);
 731             return;
 732         }
 733         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 734             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 735                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 736                          actionForNodeDefaultRoute);
 737             return;
 738         }
 739
 740         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 741                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 742         if (status.isSuccess()) {
 743             // Update cache
 744             if (actionForNodeDefaultRoute == Action.ADD) {
 745                 defaultRouteCache.add(cacheKey);
 746             } else {
 747                 defaultRouteCache.remove(cacheKey);
 748             }
 749         }
 750     }
 751
 752     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 753                                           String defaultGatewayMacAddress,
 754                                           String gatewayIp,
 755                                           Action actionForNodeDefaultRoute) {
 756         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 757         if (defaultGatewayMacAddress == null) {
 758             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 759                          gatewayIp, node, actionForNodeDefaultRoute);
 760             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 761         }
 762
 763         Status status;
 764         try {
 765             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 766             status = routingProvider == null ?
 767                      new Status(StatusCode.SUCCESS) :
 768                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 769                                                               defaultGatewayMacAddress, inetAddress,
 770                                                               actionForNodeDefaultRoute);
 771         } catch (UnknownHostException e) {
 772             status = new Status(StatusCode.BADREQUEST);
 773         }
 774
 775         if (status.isSuccess()) {
 776             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 777                          routingProvider == null ? "skipped" : "programmed",
 778                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 779         } else {
 780             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 781                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 782         }
 783         return status;
 784     }
 785
 786     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 787         Preconditions.checkNotNull(neutronFloatingIP);
 788
 789         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 790         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 791
 792         // If there is no router interface handling the networkUUID, we are done
 793         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 794             return;
 795         }
 796
 797         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 798         final String providerSegmentationId = neutronNetwork != null ?
 799                                               neutronNetwork.getProviderSegmentationID() : null;
 800         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 801         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 802
 803         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 804             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 805             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 806             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 807             return;  // done: go no further w/out all the info needed...
 808         }
 809
 810         final Action action = isDelete ? Action.DELETE : Action.ADD;
 811         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 812         if (nodes.isEmpty()) {
 813             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 814         }
 815         for (Node node : nodes) {
 816             final Long dpid = getDpidForIntegrationBridge(node);
 817             if (dpid == null) {
 818                 continue;
 819             }
 820
 821             final Action actionForNode =
 822                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 823                     action : Action.DELETE;
 824
 825             // Rewrite from float to fixed and vice-versa
 826             //
 827             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 828                                    floatingIpAddress, fixedIPAddress, actionForNode);
 829             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 830                                    fixedIPAddress, floatingIpAddress, actionForNode);
 831
 832             // Respond to arps for the floating ip address
 833             //
 834             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 835                                    actionForNode);
 836         }
 837     }
 838
 839     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 840                                         final boolean isInbound,
 841                                         String matchAddress, String rewriteAddress,
 842                                         Action actionForNode) {
 843         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 844         // will look at desired action for node.
 845         //
 846         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" +
 847                                 matchAddress + ":" + rewriteAddress;
 848         final Boolean isProgrammed = isInbound ?
 849                                      inboundIpRewriteCache.contains(cacheKey) :
 850                                      outboundIpRewriteCache.contains(cacheKey);
 851
 852         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 853             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 854                          " is already done",
 855                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 856                          matchAddress, rewriteAddress, actionForNode);
 857             return;
 858         }
 859         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 860             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 861                          " is already done",
 862                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 863                          matchAddress, rewriteAddress, actionForNode);
 864             return;
 865         }
 866
 867         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 868                                                     matchAddress, rewriteAddress, actionForNode);
 869         if (status.isSuccess()) {
 870             // Update cache
 871             if (actionForNode == Action.ADD) {
 872                 if (isInbound) {
 873                     inboundIpRewriteCache.add(cacheKey);
 874                 } else {
 875                     outboundIpRewriteCache.add(cacheKey);
 876                 }
 877             } else {
 878                 if (isInbound) {
 879                     inboundIpRewriteCache.remove(cacheKey);
 880                 } else {
 881                     outboundIpRewriteCache.remove(cacheKey);
 882                 }
 883             }
 884         }
 885     }
 886
 887     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 888                                           final boolean isInbound,
 889                                           String matchAddress, String rewriteAddress,
 890                                           Action actionForNode) {
 891         Status status;
 892         try {
 893             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 894             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 895             if (isInbound) {
 896                 status = inboundNatProvider == null ?
 897                          new Status(StatusCode.SUCCESS) :
 898                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 899                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 900             } else {
 901                 status = outboundNatProvider == null ?
 902                          new Status(StatusCode.SUCCESS) :
 903                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 904                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 905             }
 906         } catch (UnknownHostException e) {
 907             status = new Status(StatusCode.BADREQUEST);
 908         }
 909
 910         if (status.isSuccess()) {
 911             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 912             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 913                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 914                          matchAddress, rewriteAddress, node, actionForNode);
 915         } else {
 916             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 917                          (isInbound ? "inbound" : "outbound"),
 918                          matchAddress, rewriteAddress, node, actionForNode, status);
 919         }
 920         return status;
 921     }
 922
 923     private int getMaskLenFromCidr(String cidr) {
 924         if (cidr == null) return 0;
 925         String[] splits = cidr.split("/");
 926         if (splits.length != 2) return 0;
 927
 928         int result;
 929         try {
 930             result = Integer.parseInt(splits[1].trim());
 931         }
 932         catch (NumberFormatException nfe)
 933         {
 934             result = 0;
 935         }
 936         return result;
 937     }
 938
 939     private Long getDpidForIntegrationBridge(Node node) {
 940         // Check if node is integration bridge; and only then return its dpid
 941         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
 942             return southbound.getDataPathId(node);
 943         }
 944         return null;
 945     }
 946
 947     @Override
 948     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
 949         tenantNetworkManager =
 950                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
 951         configurationService =
 952                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
 953         arpProvider =
 954                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
 955         inboundNatProvider =
 956                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
 957         outboundNatProvider =
 958                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
 959         routingProvider =
 960                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
 961         l3ForwardingProvider =
 962                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
 963         nodeCacheManager =
 964                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
 965         southbound =
 966                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
 967
 968         initL3AdapterMembers();
 969     }
 970
 971     @Override
 972     public void setDependencies(Object impl) {
 973         if (impl instanceof INeutronNetworkCRUD) {
 974             neutronNetworkCache = (INeutronNetworkCRUD)impl;
 975         } else if (impl instanceof INeutronPortCRUD) {
 976             neutronPortCache = (INeutronPortCRUD)impl;
 977         } else if (impl instanceof INeutronSubnetCRUD) {
 978             neutronSubnetCache = (INeutronSubnetCRUD)impl;
 979         } else if (impl instanceof ArpProvider) {
 980             arpProvider = (ArpProvider)impl;
 981         } else if (impl instanceof InboundNatProvider) {
 982             inboundNatProvider = (InboundNatProvider)impl;
 983         } else if (impl instanceof OutboundNatProvider) {
 984             outboundNatProvider = (OutboundNatProvider)impl;
 985         } else if (impl instanceof RoutingProvider) {
 986             routingProvider = (RoutingProvider)impl;
 987         } else if (impl instanceof L3ForwardingProvider) {
 988             l3ForwardingProvider = (L3ForwardingProvider)impl;
 989         }
 990     }
 991 }