openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.controller.networkconfig.neutron.INeutronNetworkCRUD;
  14 import org.opendaylight.controller.networkconfig.neutron.INeutronPortCRUD;
  15 import org.opendaylight.controller.networkconfig.neutron.INeutronSubnetCRUD;
  16 import org.opendaylight.controller.networkconfig.neutron.NeutronFloatingIP;
  17 import org.opendaylight.controller.networkconfig.neutron.NeutronNetwork;
  18 import org.opendaylight.controller.networkconfig.neutron.NeutronPort;
  19 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter;
  20 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter_Interface;
  21 import org.opendaylight.controller.networkconfig.neutron.NeutronSubnet;
  22 import org.opendaylight.controller.networkconfig.neutron.Neutron_IPs;
  23 import org.opendaylight.controller.sal.core.Node;
  24 import org.opendaylight.controller.sal.utils.HexEncode;
  25 import org.opendaylight.controller.sal.utils.Status;
  26 import org.opendaylight.controller.sal.utils.StatusCode;
  27 import org.opendaylight.ovsdb.lib.notation.Row;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.MultiTenantAwareRouter;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  32 import org.opendaylight.ovsdb.plugin.api.OvsdbConfigurationService;
  33 import org.opendaylight.ovsdb.plugin.api.OvsdbConnectionService;
  34 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  41 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  42
  43 import com.google.common.base.Preconditions;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import java.net.InetAddress;
  48 import java.net.UnknownHostException;
  49 import java.util.HashMap;
  50 import java.util.HashSet;
  51 import java.util.List;
  52 import java.util.Map;
  53 import java.util.Set;
  54
  55 /**
  56  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  57  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  58  * as well as the multi-tenant router forwarding provider.
  59  */
  60 public class NeutronL3Adapter {
  61
  62     /**
  63      * Logger instance.
  64      */
  65     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  66
  67     // The implementation for each of these services is resolved by the OSGi Service Manager
  68     private volatile org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService configurationService;
  69     private volatile TenantNetworkManager tenantNetworkManager;
  70     private volatile NetworkingProviderManager networkingProviderManager;
  71     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  72     private volatile OvsdbConnectionService connectionService;
  73     private volatile INeutronNetworkCRUD neutronNetworkCache;
  74     private volatile INeutronSubnetCRUD neutronSubnetCache;
  75     private volatile INeutronPortCRUD neutronPortCache;
  76     private volatile MultiTenantAwareRouter multiTenantAwareRouter;
  77     private volatile L3ForwardingProvider l3ForwardingProvider;
  78     private volatile InboundNatProvider inboundNatProvider;
  79     private volatile OutboundNatProvider outboundNatProvider;
  80     private volatile ArpProvider arpProvider;
  81     private volatile RoutingProvider routingProvider;
  82
  83     private Set<String> inboundIpRewriteCache;
  84     private Set<String> outboundIpRewriteCache;
  85     private Set<String> inboundIpRewriteExclusionCache;
  86     private Set<String> outboundIpRewriteExclusionCache;
  87     private Set<String> routerInterfacesCache;
  88     private Set<String> staticArpEntryCache;
  89     private Set<String> l3ForwardingCache;
  90     private Set<String> defaultRouteCache;
  91     private Map<String, String> networkIdToRouterMacCache;
  92     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  93     private Boolean enabled = false;
  94
  95     void init() {
  96         final String enabledPropertyStr = System.getProperty("ovsdb.l3.fwd.enabled");
  97         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  98             this.inboundIpRewriteCache = new HashSet<>();
  99             this.outboundIpRewriteCache = new HashSet<>();
 100             this.inboundIpRewriteExclusionCache = new HashSet<>();
 101             this.outboundIpRewriteExclusionCache = new HashSet<>();
 102             this.routerInterfacesCache = new HashSet<>();
 103             this.staticArpEntryCache = new HashSet<>();
 104             this.l3ForwardingCache = new HashSet<>();
 105             this.defaultRouteCache = new HashSet<>();
 106             this.networkIdToRouterMacCache = new HashMap<>();
 107             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 108
 109             this.enabled = true;
 110             logger.info("OVSDB L3 forwarding is enabled");
 111         } else {
 112             logger.debug("OVSDB L3 forwarding is disabled");
 113         }
 114     }
 115
 116     //
 117     // Callbacks from OVSDB's northbound handlers
 118     //
 119
 120     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 121         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 122         if (!this.enabled)
 123             return;
 124     }
 125
 126     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 127         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 128         if (!this.enabled)
 129             return;
 130
 131         final boolean isDelete = action == Action.DELETE;
 132
 133         // Treat the port event as a router interface event if the port belongs to router. This is a
 134         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 135         //
 136         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 137             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 138                 NeutronRouter_Interface neutronRouterInterface =
 139                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 140                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 141                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 142
 143                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 144             }
 145         } else {
 146             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 147             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 148             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 149             // there.
 150             //
 151             if (!isDelete) {
 152                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 153                     NeutronRouter_Interface neutronRouterInterface =
 154                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 155                     if (neutronRouterInterface != null) {
 156                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 157                     }
 158                 }
 159             }
 160             this.updateL3ForNeutronPort(neutronPort, isDelete);
 161         }
 162     }
 163
 164     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 165         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 166         if (!this.enabled)
 167             return;
 168     }
 169
 170     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 171                                                   final NeutronRouter_Interface neutronRouterInterface,
 172                                                   Action action) {
 173         logger.debug("Router interface {} got event {}. Subnet {}",
 174                      neutronRouterInterface.getPortUUID(),
 175                      action,
 176                      neutronRouterInterface.getSubnetUUID());
 177         if (!this.enabled)
 178             return;
 179
 180         final boolean isDelete = action == Action.DELETE;
 181
 182         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 183
 184         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 185         // see if they are affected by l3
 186         //
 187         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 188             boolean currPortIsInSameSubnet = false;
 189             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 190                 if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 191                     currPortIsInSameSubnet = true;
 192                     break;
 193                 }
 194             }
 195             if (currPortIsInSameSubnet == true) {
 196                 this.updateL3ForNeutronPort(neutronPort, isDelete);
 197             }
 198         }
 199     }
 200
 201     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 202                                              Action action) {
 203         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 204                      neutronFloatingIP.getFixedIPAddress(),
 205                      neutronFloatingIP.getFloatingIPAddress(),
 206                      neutronFloatingIP.getFloatingNetworkUUID());
 207         if (!this.enabled)
 208             return;
 209
 210         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 211     }
 212
 213     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 214         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 215         if (!this.enabled)
 216             return;
 217     }
 218
 219     //
 220     // Callbacks from OVSDB's southbound handler
 221     //
 222     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 223                                      Action action) {
 224         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 225                      action, node, intf.getName(), neutronNetwork);
 226         if (!this.enabled)
 227             return;
 228
 229         // See if there is an external uuid, so we can find the respective neutronPort
 230         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 231         if (externalIds == null) {
 232             return;
 233         }
 234         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 235         if (neutronPortId == null) {
 236             return;
 237         }
 238         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 239         if (neutronPort == null) {
 240             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 241                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 242             return;
 243         }
 244         this.handleNeutronPortEvent(neutronPort, action);
 245     }
 246
 247     //
 248     // Internal helpers
 249     //
 250     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 251
 252         final String networkUUID = neutronPort.getNetworkUUID();
 253         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 254
 255         // If there is no router interface handling the networkUUID, we are done
 256         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 257             return;
 258         }
 259
 260         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 261         // router interface are handled via handleNeutronRouterInterfaceEvent.
 262         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 263             return;
 264         }
 265
 266         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 267         final String providerSegmentationId = neutronNetwork != null ?
 268                                               neutronNetwork.getProviderSegmentationID() : null;
 269         final String tenantMac = neutronPort.getMacAddress();
 270
 271         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 272             tenantMac == null || tenantMac.isEmpty()) {
 273             return;  // done: go no further w/out all the info needed...
 274         }
 275
 276         final Action action = isDelete ? Action.DELETE : Action.ADD;
 277         List<Node> nodes = connectionService.getNodes();
 278         if (nodes.isEmpty()) {
 279             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 280         }
 281         for (Node node : nodes) {
 282             final Long dpid = getDpid(node);
 283             final Action actionForNode =
 284                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 285                     action : Action.DELETE;
 286             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 287                 final String tenantIpStr = neutronIP.getIpAddress();
 288                 if (tenantIpStr.isEmpty()) {
 289                     continue;
 290                 }
 291
 292                 // Configure L3 fwd
 293                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 294
 295                 // Configure distributed ARP responder
 296                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 297             }
 298         }
 299     }
 300
 301     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 302                                            String macAddress, String ipStr,
 303                                            Action actionForNode) {
 304         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 305         // will look at desired action for node.
 306         //
 307         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 308         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 309
 310         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 311             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 312                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 313             return;
 314         }
 315         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 316             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 317                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 318             return;
 319         }
 320
 321         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 322                                                        macAddress, ipStr, actionForNode);
 323         if (status.isSuccess()) {
 324             // Update cache
 325             if (actionForNode == Action.ADD) {
 326                 l3ForwardingCache.add(cacheKey);
 327             } else {
 328                 l3ForwardingCache.remove(cacheKey);
 329             }
 330         }
 331     }
 332
 333     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 334                                              String macAddress,
 335                                              String address,
 336                                              Action actionForNode) {
 337         Status status;
 338         try {
 339             InetAddress inetAddress = InetAddress.getByName(address);
 340             status = l3ForwardingProvider == null ?
 341                      new Status(StatusCode.SUCCESS) :
 342                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 343                                                                       inetAddress, macAddress, actionForNode);
 344         } catch (UnknownHostException e) {
 345             status = new Status(StatusCode.BADREQUEST);
 346         }
 347
 348         if (status.isSuccess()) {
 349             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 350                          l3ForwardingProvider == null ? "skipped" : "programmed",
 351                          macAddress, address, node, actionForNode);
 352         } else {
 353             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 354                          macAddress, address, node, actionForNode, status);
 355         }
 356         return status;
 357     }
 358
 359     // --
 360
 361     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface neutronRouterInterface,
 362                                                        Boolean isDelete) {
 363         Preconditions.checkNotNull(neutronRouterInterface);
 364
 365         final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
 366         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 367         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 368         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(neutronRouterInterface.getSubnetUUID());
 369         final NeutronNetwork neutronNetwork = subnet != null ?
 370                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 371         final String providerSegmentationId = neutronNetwork != null ?
 372                                               neutronNetwork.getProviderSegmentationID() : null;
 373         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 374         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 375         final String cidr = subnet != null ? subnet.getCidr() : null;
 376         final int mask = getMaskLenFromCidr(cidr);
 377
 378         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 379                      neutronRouterInterface, isDelete);
 380
 381         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 382             cidr == null || cidr.isEmpty() ||
 383             macAddress == null || macAddress.isEmpty() ||
 384             ipList == null || ipList.isEmpty()) {
 385             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 386                          providerSegmentationId, cidr, macAddress, ipList);
 387             return;  // done: go no further w/out all the info needed...
 388         }
 389
 390         final Action action = isDelete ? Action.DELETE : Action.ADD;
 391
 392         // Keep cache for finding router's mac from network uuid
 393         //
 394         if (isDelete) {
 395             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 396             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 397         } else {
 398             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 399             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), neutronRouterInterface);
 400         }
 401
 402         List<Node> nodes = connectionService.getNodes();
 403         if (nodes.isEmpty()) {
 404             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 405         }
 406         for (Node node : nodes) {
 407             final Long dpid = getDpid(node);
 408             final Action actionForNode =
 409                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 410                     action : Action.DELETE;
 411
 412             for (Neutron_IPs neutronIP : ipList) {
 413                 final String ipStr = neutronIP.getIpAddress();
 414                 if (ipStr.isEmpty()) {
 415                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 416                                  node.getID(), ipStr);
 417                     continue;
 418                 }
 419                 programRouterInterfaceStage1(node, dpid, providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 420                 programStaticArpStage1(node, dpid, providerSegmentationId, macAddress, ipStr, actionForNode);
 421             }
 422
 423             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 424             // for the external neutron networks.
 425             //
 426             {
 427                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 428                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 429                                                 cidr, actionForRewriteExclusion);
 430                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, false /* isInbound */,
 431                                                 cidr, actionForRewriteExclusion);
 432             }
 433
 434             // Default route. For non-external subnet, make sure that there is none configured.
 435             //
 436             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 437                 final Action actionForNodeDefaultRoute =
 438                         isExternal ? actionForNode : Action.DELETE;
 439                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 440                 programDefaultRouteStage1(node, dpid, providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 441                                           actionForNodeDefaultRoute);
 442             }
 443         }
 444     }
 445
 446     private void programRouterInterfaceStage1(Node node, Long dpid, String providerSegmentationId,
 447                                               String macAddress, String ipStr, int mask,
 448                                               Action actionForNode) {
 449         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 450         // will look at desired action for node.
 451         //
 452         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 453                                 ipStr + "/" + Integer.toString(mask);
 454         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 455
 456         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 457             logger.trace("programRouterInterfaceStage1 for node {} providerId {} mac {} ip {} mask {} action {}" +
 458                          " is already done",
 459                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 460             return;
 461         }
 462         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 463             logger.trace("programRouterInterfaceStage1 for node {} providerId {} mac {} ip {} mask {} action {}" +
 464                          " is already done",
 465                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 466             return;
 467         }
 468
 469         Status status = this.programRouterInterfaceStage2(node, dpid, providerSegmentationId,
 470                                                           macAddress, ipStr, mask, actionForNode);
 471         if (status.isSuccess()) {
 472             // Update cache
 473             if (actionForNode == Action.ADD) {
 474                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 475                 routerInterfacesCache.add(cacheKey);
 476             } else {
 477                 // TODO: multiTenantAwareRouter.removeInterface(...);
 478                 routerInterfacesCache.remove(cacheKey);
 479             }
 480         }
 481     }
 482
 483     private Status programRouterInterfaceStage2(Node node, Long dpid, String providerSegmentationId,
 484                                                 String macAddress,
 485                                                 String address, int mask,
 486                                                 Action actionForNode) {
 487         Status status;
 488         try {
 489             InetAddress inetAddress = InetAddress.getByName(address);
 490             status = routingProvider == null ?
 491                      new Status(StatusCode.SUCCESS) :
 492                      routingProvider.programRouterInterface(node, dpid, providerSegmentationId,
 493                                                             macAddress, inetAddress, mask, actionForNode);
 494         } catch (UnknownHostException e) {
 495             status = new Status(StatusCode.BADREQUEST);
 496         }
 497
 498         if (status.isSuccess()) {
 499             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 500                          routingProvider == null ? "skipped" : "programmed",
 501                          macAddress, address, mask, node, actionForNode);
 502         } else {
 503             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 504                          macAddress, address, mask, node, actionForNode, status);
 505         }
 506         return status;
 507     }
 508
 509     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 510                                         String macAddress, String ipStr,
 511                                         Action actionForNode) {
 512         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 513         // will look at desired action for node.
 514         //
 515         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 516         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 517
 518         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 519             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 520                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 521             return;
 522         }
 523         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 524             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 525                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 526             return;
 527         }
 528
 529         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 530                                                     macAddress, ipStr, actionForNode);
 531         if (status.isSuccess()) {
 532             // Update cache
 533             if (actionForNode == Action.ADD) {
 534                 staticArpEntryCache.add(cacheKey);
 535             } else {
 536                 staticArpEntryCache.remove(cacheKey);
 537             }
 538         }
 539     }
 540
 541     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 542                                                 String macAddress,
 543                                                 String address,
 544                                                 Action actionForNode) {
 545         Status status;
 546         try {
 547             InetAddress inetAddress = InetAddress.getByName(address);
 548             status = arpProvider == null ?
 549                      new Status(StatusCode.SUCCESS) :
 550                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 551                                                        macAddress, inetAddress, actionForNode);
 552         } catch (UnknownHostException e) {
 553             status = new Status(StatusCode.BADREQUEST);
 554         }
 555
 556         if (status.isSuccess()) {
 557             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 558                          arpProvider == null ? "skipped" : "programmed",
 559                          macAddress, address, node, actionForNode);
 560         } else {
 561             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 562                          macAddress, address, node, actionForNode, status);
 563         }
 564         return status;
 565     }
 566
 567     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 568                                                  final boolean isInbound, String cidr,
 569                                                  Action actionForRewriteExclusion) {
 570         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 571         // will look at desired action for node.
 572         //
 573         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 574         final Boolean isProgrammed = isInbound ?
 575                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 576                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 577
 578         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 579             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 580                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 581                          actionForRewriteExclusion);
 582             return;
 583         }
 584         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 585             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 586                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 587                          actionForRewriteExclusion);
 588             return;
 589         }
 590
 591         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 592                                                              isInbound, actionForRewriteExclusion);
 593         if (status.isSuccess()) {
 594             // Update cache
 595             if (actionForRewriteExclusion == Action.ADD) {
 596                 if (isInbound) {
 597                     inboundIpRewriteExclusionCache.add(cacheKey);
 598                 } else {
 599                     outboundIpRewriteExclusionCache.add(cacheKey);
 600                 }
 601             } else {
 602                 if (isInbound) {
 603                     inboundIpRewriteExclusionCache.remove(cacheKey);
 604                 } else {
 605                     outboundIpRewriteExclusionCache.remove(cacheKey);
 606                 }
 607             }
 608         }
 609     }
 610
 611     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 612                                                    final boolean isInbound, Action actionForNode) {
 613         Status status;
 614         if (isInbound) {
 615             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 616                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 617                                                                   actionForNode);
 618         } else {
 619             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 620                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 621                                                                    actionForNode);
 622         }
 623
 624         if (status.isSuccess()) {
 625             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 626             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 627                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 628                          cidr, node, actionForNode);
 629         } else {
 630             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 631                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 632         }
 633         return status;
 634     }
 635
 636     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 637                                            String defaultGatewayMacAddress, String gatewayIp,
 638                                            Action actionForNodeDefaultRoute) {
 639         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 640         // will look at desired action for node.
 641         //
 642         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 643         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 644
 645         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 646             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 647                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 648                          actionForNodeDefaultRoute);
 649             return;
 650         }
 651         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 652             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 653                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 654                          actionForNodeDefaultRoute);
 655             return;
 656         }
 657
 658         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 659                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 660         if (status.isSuccess()) {
 661             // Update cache
 662             if (actionForNodeDefaultRoute == Action.ADD) {
 663                 defaultRouteCache.add(cacheKey);
 664             } else {
 665                 defaultRouteCache.remove(cacheKey);
 666             }
 667         }
 668     }
 669
 670     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 671                                           String defaultGatewayMacAddress,
 672                                           String gatewayIp,
 673                                           Action actionForNodeDefaultRoute) {
 674         Status status;
 675         try {
 676             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 677             status = routingProvider == null ?
 678                      new Status(StatusCode.SUCCESS) :
 679                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 680                                                               defaultGatewayMacAddress, inetAddress,
 681                                                               actionForNodeDefaultRoute);
 682         } catch (UnknownHostException e) {
 683             status = new Status(StatusCode.BADREQUEST);
 684         }
 685
 686         if (status.isSuccess()) {
 687             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 688                          routingProvider == null ? "skipped" : "programmed",
 689                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 690         } else {
 691             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 692                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 693         }
 694         return status;
 695     }
 696
 697     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 698         Preconditions.checkNotNull(neutronFloatingIP);
 699
 700         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 701         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 702
 703         // If there is no router interface handling the networkUUID, we are done
 704         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 705             return;
 706         }
 707
 708         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 709         final String providerSegmentationId = neutronNetwork != null ?
 710                                               neutronNetwork.getProviderSegmentationID() : null;
 711         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 712         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 713
 714         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 715             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 716             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 717             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 718             return;  // done: go no further w/out all the info needed...
 719         }
 720
 721         final Action action = isDelete ? Action.DELETE : Action.ADD;
 722         List<Node> nodes = connectionService.getNodes();
 723         if (nodes.isEmpty()) {
 724             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 725         }
 726         for (Node node : nodes) {
 727             final Long dpid = getDpid(node);
 728             final Action actionForNode =
 729                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 730                     action : Action.DELETE;
 731
 732             // Rewrite from float to fixed and vice-versa
 733             //
 734             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 735                                    floatingIpAddress, fixedIPAddress, actionForNode);
 736             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 737                                    fixedIPAddress, floatingIpAddress, actionForNode);
 738
 739             // Respond to arps for the floating ip address
 740             //
 741             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 742                                    actionForNode);
 743         }
 744     }
 745
 746     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 747                                         final boolean isInbound,
 748                                         String matchAddress, String rewriteAddress,
 749                                         Action actionForNode) {
 750         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 751         // will look at desired action for node.
 752         //
 753         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 754                                 matchAddress + ":" + rewriteAddress;
 755         final Boolean isProgrammed = isInbound ?
 756                                      inboundIpRewriteCache.contains(cacheKey) :
 757                                      outboundIpRewriteCache.contains(cacheKey);
 758
 759         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 760             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 761                          " is already done",
 762                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 763                          matchAddress, rewriteAddress, actionForNode);
 764             return;
 765         }
 766         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 767             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 768                          " is already done",
 769                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 770                          matchAddress, rewriteAddress, actionForNode);
 771             return;
 772         }
 773
 774         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 775                                                     matchAddress, rewriteAddress, actionForNode);
 776         if (status.isSuccess()) {
 777             // Update cache
 778             if (actionForNode == Action.ADD) {
 779                 if (isInbound) {
 780                     inboundIpRewriteCache.add(cacheKey);
 781                 } else {
 782                     outboundIpRewriteCache.add(cacheKey);
 783                 }
 784             } else {
 785                 if (isInbound) {
 786                     inboundIpRewriteCache.remove(cacheKey);
 787                 } else {
 788                     outboundIpRewriteCache.remove(cacheKey);
 789                 }
 790             }
 791         }
 792     }
 793
 794     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 795                                           final boolean isInbound,
 796                                           String matchAddress, String rewriteAddress,
 797                                           Action actionForNode) {
 798         Status status;
 799         try {
 800             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 801             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 802             if (isInbound) {
 803                 status = inboundNatProvider == null ?
 804                          new Status(StatusCode.SUCCESS) :
 805                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 806                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 807             } else {
 808                 status = outboundNatProvider == null ?
 809                          new Status(StatusCode.SUCCESS) :
 810                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 811                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 812             }
 813         } catch (UnknownHostException e) {
 814             status = new Status(StatusCode.BADREQUEST);
 815         }
 816
 817         if (status.isSuccess()) {
 818             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 819             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 820                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 821                          matchAddress, rewriteAddress, node, actionForNode);
 822         } else {
 823             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 824                          (isInbound ? "inbound" : "outbound"),
 825                          matchAddress, rewriteAddress, node, actionForNode, status);
 826         }
 827         return status;
 828     }
 829
 830     //
 831     // More Internals
 832     //
 833
 834     private int getMaskLenFromCidr(String cidr) {
 835         if (cidr == null) return 0;
 836         String[] splits = cidr.split("/");
 837         if (splits.length != 2) return 0;
 838
 839         int result;
 840         try {
 841             result = Integer.parseInt(splits[1].trim());
 842         }
 843         catch (NumberFormatException nfe)
 844         {
 845             result = 0;
 846         }
 847         return result;
 848     }
 849
 850     private Long getDpid (Node node) {
 851         Preconditions.checkNotNull(ovsdbConfigurationService);
 852
 853         String bridgeName = configurationService.getIntegrationBridgeName();
 854         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 855         if (bridgeUuid == null) {
 856             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 857             return 0L;
 858         }
 859
 860         try {
 861             Row bridgeRow =  ovsdbConfigurationService
 862                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 863             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 864             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 865             if (dpids == null || dpids.size() == 0) return 0L;
 866             return HexEncode.stringToLong((String) dpids.toArray()[0]);
 867         } catch (Exception e) {
 868             logger.error("Error finding Bridge's OF DPID", e);
 869             return 0L;
 870         }
 871     }
 872
 873     private String getInternalBridgeUUID (Node node, String bridgeName) {
 874         Preconditions.checkNotNull(ovsdbConfigurationService);
 875         try {
 876             Map<String, Row> bridgeTable =
 877                     ovsdbConfigurationService.getRows(node,
 878                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 879             if (bridgeTable == null) return null;
 880             for (String key : bridgeTable.keySet()) {
 881                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 882                 if (bridge.getName().equals(bridgeName)) return key;
 883             }
 884         } catch (Exception e) {
 885             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 886         }
 887         return null;
 888     }
 889 }
 890
 891