openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.controller.networkconfig.neutron.INeutronNetworkCRUD;
  14 import org.opendaylight.controller.networkconfig.neutron.INeutronPortCRUD;
  15 import org.opendaylight.controller.networkconfig.neutron.INeutronSubnetCRUD;
  16 import org.opendaylight.controller.networkconfig.neutron.NeutronFloatingIP;
  17 import org.opendaylight.controller.networkconfig.neutron.NeutronNetwork;
  18 import org.opendaylight.controller.networkconfig.neutron.NeutronPort;
  19 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter;
  20 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter_Interface;
  21 import org.opendaylight.controller.networkconfig.neutron.NeutronSubnet;
  22 import org.opendaylight.controller.networkconfig.neutron.Neutron_IPs;
  23 import org.opendaylight.controller.sal.core.Node;
  24 import org.opendaylight.controller.sal.utils.HexEncode;
  25 import org.opendaylight.controller.sal.utils.Status;
  26 import org.opendaylight.controller.sal.utils.StatusCode;
  27 import org.opendaylight.ovsdb.lib.notation.Row;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.MultiTenantAwareRouter;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  32 import org.opendaylight.ovsdb.plugin.api.OvsdbConfigurationService;
  33 import org.opendaylight.ovsdb.plugin.api.OvsdbConnectionService;
  34 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  41 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  42
  43 import com.google.common.base.Preconditions;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import java.net.InetAddress;
  48 import java.net.UnknownHostException;
  49 import java.util.HashMap;
  50 import java.util.HashSet;
  51 import java.util.List;
  52 import java.util.Map;
  53 import java.util.Set;
  54
  55 /**
  56  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  57  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  58  * as well as the multi-tenant router forwarding provider.
  59  */
  60 public class NeutronL3Adapter {
  61
  62     /**
  63      * Logger instance.
  64      */
  65     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  66
  67     // The implementation for each of these services is resolved by the OSGi Service Manager
  68     private volatile org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService configurationService;
  69     private volatile TenantNetworkManager tenantNetworkManager;
  70     private volatile NetworkingProviderManager networkingProviderManager;
  71     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  72     private volatile OvsdbConnectionService connectionService;
  73     private volatile INeutronNetworkCRUD neutronNetworkCache;
  74     private volatile INeutronSubnetCRUD neutronSubnetCache;
  75     private volatile INeutronPortCRUD neutronPortCache;
  76     private volatile MultiTenantAwareRouter multiTenantAwareRouter;
  77     private volatile L3ForwardingProvider l3ForwardingProvider;
  78     private volatile InboundNatProvider inboundNatProvider;
  79     private volatile OutboundNatProvider outboundNatProvider;
  80     private volatile ArpProvider arpProvider;
  81     private volatile RoutingProvider routingProvider;
  82
  83     private Set<String> inboundIpRewriteCache;
  84     private Set<String> outboundIpRewriteCache;
  85     private Set<String> inboundIpRewriteExclusionCache;
  86     private Set<String> outboundIpRewriteExclusionCache;
  87     private Set<String> routerInterfacesCache;
  88     private Set<String> staticArpEntryCache;
  89     private Set<String> l3ForwardingCache;
  90     private Set<String> defaultRouteCache;
  91     private Map<String, String> networkIdToRouterMacCache;
  92     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  93
  94     void init() {
  95         this.inboundIpRewriteCache = new HashSet<>();
  96         this.outboundIpRewriteCache = new HashSet<>();
  97         this.inboundIpRewriteExclusionCache = new HashSet<>();
  98         this.outboundIpRewriteExclusionCache = new HashSet<>();
  99         this.routerInterfacesCache = new HashSet<>();
 100         this.staticArpEntryCache = new HashSet<>();
 101         this.l3ForwardingCache = new HashSet<>();
 102         this.defaultRouteCache = new HashSet<>();
 103         this.networkIdToRouterMacCache = new HashMap<>();
 104         this.subnetIdToRouterInterfaceCache = new HashMap<>();
 105     }
 106
 107     //
 108     // Callbacks from OVSDB's northbound handlers
 109     //
 110
 111     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 112         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 113     }
 114
 115     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 116         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 117
 118         final boolean isDelete = action == Action.DELETE;
 119
 120         // Treat the port event as a router interface event if the port belongs to router. This is a
 121         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 122         //
 123         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 124             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 125                 NeutronRouter_Interface neutronRouterInterface =
 126                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 127                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 128                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 129
 130                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 131             }
 132         } else {
 133             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 134             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 135             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 136             // there.
 137             //
 138             if (!isDelete) {
 139                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 140                     NeutronRouter_Interface neutronRouterInterface =
 141                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 142                     if (neutronRouterInterface != null) {
 143                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 144                     }
 145                 }
 146             }
 147             this.updateL3ForNeutronPort(neutronPort, isDelete);
 148         }
 149     }
 150
 151     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 152         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 153     }
 154
 155     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 156                                                   final NeutronRouter_Interface neutronRouterInterface,
 157                                                   Action action) {
 158         logger.debug("Router interface {} got event {}. Subnet {}",
 159                      neutronRouterInterface.getPortUUID(),
 160                      action,
 161                      neutronRouterInterface.getSubnetUUID());
 162
 163         final boolean isDelete = action == Action.DELETE;
 164
 165         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 166
 167         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 168         // see if they are affected by l3
 169         //
 170         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 171             boolean currPortIsInSameSubnet = false;
 172             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 173                 if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 174                     currPortIsInSameSubnet = true;
 175                     break;
 176                 }
 177             }
 178             if (currPortIsInSameSubnet == true) {
 179                 this.updateL3ForNeutronPort(neutronPort, isDelete);
 180             }
 181         }
 182     }
 183
 184     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 185                                              Action action) {
 186         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 187                      neutronFloatingIP.getFixedIPAddress(),
 188                      neutronFloatingIP.getFloatingIPAddress(),
 189                      neutronFloatingIP.getFloatingNetworkUUID());
 190
 191         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 192     }
 193
 194     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 195         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 196     }
 197
 198     //
 199     // Callbacks from OVSDB's southbound handler
 200     //
 201     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 202                                      Action action) {
 203         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 204                      action, node, intf.getName(), neutronNetwork);
 205
 206         // See if there is an external uuid, so we can find the respective neutronPort
 207         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 208         if (externalIds == null) {
 209             return;
 210         }
 211         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 212         if (neutronPortId == null) {
 213             return;
 214         }
 215         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 216         if (neutronPort == null) {
 217             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 218                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 219             return;
 220         }
 221         this.handleNeutronPortEvent(neutronPort, action);
 222     }
 223
 224     //
 225     // Internal helpers
 226     //
 227     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 228
 229         final String networkUUID = neutronPort.getNetworkUUID();
 230         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 231
 232         // If there is no router interface handling the networkUUID, we are done
 233         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 234             return;
 235         }
 236
 237         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 238         // router interface are handled via handleNeutronRouterInterfaceEvent.
 239         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 240             return;
 241         }
 242
 243         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 244         final String providerSegmentationId = neutronNetwork != null ?
 245                                               neutronNetwork.getProviderSegmentationID() : null;
 246         final String tenantMac = neutronPort.getMacAddress();
 247
 248         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 249             tenantMac == null || tenantMac.isEmpty()) {
 250             return;  // done: go no further w/out all the info needed...
 251         }
 252
 253         final Action action = isDelete ? Action.DELETE : Action.ADD;
 254         List<Node> nodes = connectionService.getNodes();
 255         if (nodes.isEmpty()) {
 256             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 257         }
 258         for (Node node : nodes) {
 259             final Long dpid = getDpid(node);
 260             final Action actionForNode =
 261                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 262                     action : Action.DELETE;
 263             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 264                 final String tenantIpStr = neutronIP.getIpAddress();
 265                 if (tenantIpStr.isEmpty()) {
 266                     continue;
 267                 }
 268
 269                 // Configure L3 fwd
 270                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 271
 272                 // Configure distributed ARP responder
 273                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 274             }
 275         }
 276     }
 277
 278     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 279                                            String macAddress, String ipStr,
 280                                            Action actionForNode) {
 281         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 282         // will look at desired action for node.
 283         //
 284         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 285         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 286
 287         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 288             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 289                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 290             return;
 291         }
 292         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 293             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 294                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 295             return;
 296         }
 297
 298         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 299                                                        macAddress, ipStr, actionForNode);
 300         if (status.isSuccess()) {
 301             // Update cache
 302             if (actionForNode == Action.ADD) {
 303                 l3ForwardingCache.add(cacheKey);
 304             } else {
 305                 l3ForwardingCache.remove(cacheKey);
 306             }
 307         }
 308     }
 309
 310     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 311                                              String macAddress,
 312                                              String address,
 313                                              Action actionForNode) {
 314         Status status;
 315         try {
 316             InetAddress inetAddress = InetAddress.getByName(address);
 317             status = l3ForwardingProvider == null ?
 318                      new Status(StatusCode.SUCCESS) :
 319                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 320                                                                       inetAddress, macAddress, actionForNode);
 321         } catch (UnknownHostException e) {
 322             status = new Status(StatusCode.BADREQUEST);
 323         }
 324
 325         if (status.isSuccess()) {
 326             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 327                          l3ForwardingProvider == null ? "skipped" : "programmed",
 328                          macAddress, address, node, actionForNode);
 329         } else {
 330             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 331                          macAddress, address, node, actionForNode, status);
 332         }
 333         return status;
 334     }
 335
 336     // --
 337
 338     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface neutronRouterInterface,
 339                                                        Boolean isDelete) {
 340         Preconditions.checkNotNull(neutronRouterInterface);
 341
 342         final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
 343         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 344         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 345         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(neutronRouterInterface.getSubnetUUID());
 346         final NeutronNetwork neutronNetwork = subnet != null ?
 347                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 348         final String providerSegmentationId = neutronNetwork != null ?
 349                                               neutronNetwork.getProviderSegmentationID() : null;
 350         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 351         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 352         final String cidr = subnet != null ? subnet.getCidr() : null;
 353         final int mask = getMaskLenFromCidr(cidr);
 354
 355         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 356                      neutronRouterInterface, isDelete);
 357
 358         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 359             cidr == null || cidr.isEmpty() ||
 360             macAddress == null || macAddress.isEmpty() ||
 361             ipList == null || ipList.isEmpty()) {
 362             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 363                          providerSegmentationId, cidr, macAddress, ipList);
 364             return;  // done: go no further w/out all the info needed...
 365         }
 366
 367         final Action action = isDelete ? Action.DELETE : Action.ADD;
 368
 369         // Keep cache for finding router's mac from network uuid
 370         //
 371         if (isDelete) {
 372             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 373             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 374         } else {
 375             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 376             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), neutronRouterInterface);
 377         }
 378
 379         List<Node> nodes = connectionService.getNodes();
 380         if (nodes.isEmpty()) {
 381             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 382         }
 383         for (Node node : nodes) {
 384             final Long dpid = getDpid(node);
 385             final Action actionForNode =
 386                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 387                     action : Action.DELETE;
 388
 389             for (Neutron_IPs neutronIP : ipList) {
 390                 final String ipStr = neutronIP.getIpAddress();
 391                 if (ipStr.isEmpty()) {
 392                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 393                                  node.getID(), ipStr);
 394                     continue;
 395                 }
 396                 programRouterInterfaceStage1(node, dpid, providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 397                 programStaticArpStage1(node, dpid, providerSegmentationId, macAddress, ipStr, actionForNode);
 398             }
 399
 400             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 401             // for the external neutron networks.
 402             //
 403             {
 404                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 405                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 406                                                 cidr, actionForRewriteExclusion);
 407                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, false /* isInbound */,
 408                                                 cidr, actionForRewriteExclusion);
 409             }
 410
 411             // Default route. For non-external subnet, make sure that there is none configured.
 412             //
 413             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 414                 final Action actionForNodeDefaultRoute =
 415                         isExternal ? actionForNode : Action.DELETE;
 416                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 417                 programDefaultRouteStage1(node, dpid, providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 418                                           actionForNodeDefaultRoute);
 419             }
 420         }
 421     }
 422
 423     private void programRouterInterfaceStage1(Node node, Long dpid, String providerSegmentationId,
 424                                               String macAddress, String ipStr, int mask,
 425                                               Action actionForNode) {
 426         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 427         // will look at desired action for node.
 428         //
 429         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 430                                 ipStr + "/" + Integer.toString(mask);
 431         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 432
 433         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 434             logger.trace("programRouterInterfaceStage1 for node {} providerId {} mac {} ip {} mask {} action {}" +
 435                          " is already done",
 436                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 437             return;
 438         }
 439         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 440             logger.trace("programRouterInterfaceStage1 for node {} providerId {} mac {} ip {} mask {} action {}" +
 441                          " is already done",
 442                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 443             return;
 444         }
 445
 446         Status status = this.programRouterInterfaceStage2(node, dpid, providerSegmentationId,
 447                                                           macAddress, ipStr, mask, actionForNode);
 448         if (status.isSuccess()) {
 449             // Update cache
 450             if (actionForNode == Action.ADD) {
 451                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 452                 routerInterfacesCache.add(cacheKey);
 453             } else {
 454                 // TODO: multiTenantAwareRouter.removeInterface(...);
 455                 routerInterfacesCache.remove(cacheKey);
 456             }
 457         }
 458     }
 459
 460     private Status programRouterInterfaceStage2(Node node, Long dpid, String providerSegmentationId,
 461                                                 String macAddress,
 462                                                 String address, int mask,
 463                                                 Action actionForNode) {
 464         Status status;
 465         try {
 466             InetAddress inetAddress = InetAddress.getByName(address);
 467             status = routingProvider == null ?
 468                      new Status(StatusCode.SUCCESS) :
 469                      routingProvider.programRouterInterface(node, dpid, providerSegmentationId,
 470                                                             macAddress, inetAddress, mask, actionForNode);
 471         } catch (UnknownHostException e) {
 472             status = new Status(StatusCode.BADREQUEST);
 473         }
 474
 475         if (status.isSuccess()) {
 476             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 477                          routingProvider == null ? "skipped" : "programmed",
 478                          macAddress, address, mask, node, actionForNode);
 479         } else {
 480             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 481                          macAddress, address, mask, node, actionForNode, status);
 482         }
 483         return status;
 484     }
 485
 486     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 487                                         String macAddress, String ipStr,
 488                                         Action actionForNode) {
 489         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 490         // will look at desired action for node.
 491         //
 492         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 493         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 494
 495         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 496             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 497                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 498             return;
 499         }
 500         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 501             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 502                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 503             return;
 504         }
 505
 506         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 507                                                     macAddress, ipStr, actionForNode);
 508         if (status.isSuccess()) {
 509             // Update cache
 510             if (actionForNode == Action.ADD) {
 511                 staticArpEntryCache.add(cacheKey);
 512             } else {
 513                 staticArpEntryCache.remove(cacheKey);
 514             }
 515         }
 516     }
 517
 518     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 519                                                 String macAddress,
 520                                                 String address,
 521                                                 Action actionForNode) {
 522         Status status;
 523         try {
 524             InetAddress inetAddress = InetAddress.getByName(address);
 525             status = arpProvider == null ?
 526                      new Status(StatusCode.SUCCESS) :
 527                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 528                                                        macAddress, inetAddress, actionForNode);
 529         } catch (UnknownHostException e) {
 530             status = new Status(StatusCode.BADREQUEST);
 531         }
 532
 533         if (status.isSuccess()) {
 534             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 535                          arpProvider == null ? "skipped" : "programmed",
 536                          macAddress, address, node, actionForNode);
 537         } else {
 538             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 539                          macAddress, address, node, actionForNode, status);
 540         }
 541         return status;
 542     }
 543
 544     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 545                                                  final boolean isInbound, String cidr,
 546                                                  Action actionForRewriteExclusion) {
 547         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 548         // will look at desired action for node.
 549         //
 550         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 551         final Boolean isProgrammed = isInbound ?
 552                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 553                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 554
 555         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 556             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 557                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 558                          actionForRewriteExclusion);
 559             return;
 560         }
 561         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 562             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 563                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 564                          actionForRewriteExclusion);
 565             return;
 566         }
 567
 568         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 569                                                              isInbound, actionForRewriteExclusion);
 570         if (status.isSuccess()) {
 571             // Update cache
 572             if (actionForRewriteExclusion == Action.ADD) {
 573                 if (isInbound) {
 574                     inboundIpRewriteExclusionCache.add(cacheKey);
 575                 } else {
 576                     outboundIpRewriteExclusionCache.add(cacheKey);
 577                 }
 578             } else {
 579                 if (isInbound) {
 580                     inboundIpRewriteExclusionCache.remove(cacheKey);
 581                 } else {
 582                     outboundIpRewriteExclusionCache.remove(cacheKey);
 583                 }
 584             }
 585         }
 586     }
 587
 588     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 589                                                    final boolean isInbound, Action actionForNode) {
 590         Status status;
 591         if (isInbound) {
 592             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 593                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 594                                                                   actionForNode);
 595         } else {
 596             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 597                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 598                                                                    actionForNode);
 599         }
 600
 601         if (status.isSuccess()) {
 602             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 603             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 604                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 605                          cidr, node, actionForNode);
 606         } else {
 607             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 608                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 609         }
 610         return status;
 611     }
 612
 613     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 614                                            String defaultGatewayMacAddress, String gatewayIp,
 615                                            Action actionForNodeDefaultRoute) {
 616         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 617         // will look at desired action for node.
 618         //
 619         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 620         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 621
 622         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 623             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 624                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 625                          actionForNodeDefaultRoute);
 626             return;
 627         }
 628         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 629             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 630                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 631                          actionForNodeDefaultRoute);
 632             return;
 633         }
 634
 635         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 636                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 637         if (status.isSuccess()) {
 638             // Update cache
 639             if (actionForNodeDefaultRoute == Action.ADD) {
 640                 defaultRouteCache.add(cacheKey);
 641             } else {
 642                 defaultRouteCache.remove(cacheKey);
 643             }
 644         }
 645     }
 646
 647     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 648                                           String defaultGatewayMacAddress,
 649                                           String gatewayIp,
 650                                           Action actionForNodeDefaultRoute) {
 651         Status status;
 652         try {
 653             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 654             status = routingProvider == null ?
 655                      new Status(StatusCode.SUCCESS) :
 656                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 657                                                               defaultGatewayMacAddress, inetAddress,
 658                                                               actionForNodeDefaultRoute);
 659         } catch (UnknownHostException e) {
 660             status = new Status(StatusCode.BADREQUEST);
 661         }
 662
 663         if (status.isSuccess()) {
 664             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 665                          routingProvider == null ? "skipped" : "programmed",
 666                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 667         } else {
 668             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 669                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 670         }
 671         return status;
 672     }
 673
 674     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 675         Preconditions.checkNotNull(neutronFloatingIP);
 676
 677         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 678         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 679
 680         // If there is no router interface handling the networkUUID, we are done
 681         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 682             return;
 683         }
 684
 685         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 686         final String providerSegmentationId = neutronNetwork != null ?
 687                                               neutronNetwork.getProviderSegmentationID() : null;
 688         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 689         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 690
 691         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 692             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 693             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 694             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 695             return;  // done: go no further w/out all the info needed...
 696         }
 697
 698         final Action action = isDelete ? Action.DELETE : Action.ADD;
 699         List<Node> nodes = connectionService.getNodes();
 700         if (nodes.isEmpty()) {
 701             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 702         }
 703         for (Node node : nodes) {
 704             final Long dpid = getDpid(node);
 705             final Action actionForNode =
 706                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 707                     action : Action.DELETE;
 708
 709             // Rewrite from float to fixed and vice-versa
 710             //
 711             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 712                                    floatingIpAddress, fixedIPAddress, actionForNode);
 713             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 714                                    fixedIPAddress, floatingIpAddress, actionForNode);
 715
 716             // Respond to arps for the floating ip address
 717             //
 718             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 719                                    actionForNode);
 720         }
 721     }
 722
 723     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 724                                         final boolean isInbound,
 725                                         String matchAddress, String rewriteAddress,
 726                                         Action actionForNode) {
 727         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 728         // will look at desired action for node.
 729         //
 730         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 731                                 matchAddress + ":" + rewriteAddress;
 732         final Boolean isProgrammed = isInbound ?
 733                                      inboundIpRewriteCache.contains(cacheKey) :
 734                                      outboundIpRewriteCache.contains(cacheKey);
 735
 736         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 737             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 738                          " is already done",
 739                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 740                          matchAddress, rewriteAddress, actionForNode);
 741             return;
 742         }
 743         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 744             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 745                          " is already done",
 746                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 747                          matchAddress, rewriteAddress, actionForNode);
 748             return;
 749         }
 750
 751         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 752                                                     matchAddress, rewriteAddress, actionForNode);
 753         if (status.isSuccess()) {
 754             // Update cache
 755             if (actionForNode == Action.ADD) {
 756                 if (isInbound) {
 757                     inboundIpRewriteCache.add(cacheKey);
 758                 } else {
 759                     outboundIpRewriteCache.add(cacheKey);
 760                 }
 761             } else {
 762                 if (isInbound) {
 763                     inboundIpRewriteCache.remove(cacheKey);
 764                 } else {
 765                     outboundIpRewriteCache.remove(cacheKey);
 766                 }
 767             }
 768         }
 769     }
 770
 771     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 772                                           final boolean isInbound,
 773                                           String matchAddress, String rewriteAddress,
 774                                           Action actionForNode) {
 775         Status status;
 776         try {
 777             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 778             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 779             if (isInbound) {
 780                 status = inboundNatProvider == null ?
 781                          new Status(StatusCode.SUCCESS) :
 782                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 783                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 784             } else {
 785                 status = outboundNatProvider == null ?
 786                          new Status(StatusCode.SUCCESS) :
 787                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 788                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 789             }
 790         } catch (UnknownHostException e) {
 791             status = new Status(StatusCode.BADREQUEST);
 792         }
 793
 794         if (status.isSuccess()) {
 795             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 796             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 797                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 798                          matchAddress, rewriteAddress, node, actionForNode);
 799         } else {
 800             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 801                          (isInbound ? "inbound" : "outbound"),
 802                          matchAddress, rewriteAddress, node, actionForNode, status);
 803         }
 804         return status;
 805     }
 806
 807     //
 808     // More Internals
 809     //
 810
 811     private int getMaskLenFromCidr(String cidr) {
 812         if (cidr == null) return 0;
 813         String[] splits = cidr.split("/");
 814         if (splits.length != 2) return 0;
 815
 816         int result;
 817         try {
 818             result = Integer.parseInt(splits[1].trim());
 819         }
 820         catch (NumberFormatException nfe)
 821         {
 822             result = 0;
 823         }
 824         return result;
 825     }
 826
 827     private Long getDpid (Node node) {
 828         Preconditions.checkNotNull(ovsdbConfigurationService);
 829
 830         String bridgeName = configurationService.getIntegrationBridgeName();
 831         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 832         if (bridgeUuid == null) {
 833             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 834             return 0L;
 835         }
 836
 837         try {
 838             Row bridgeRow =  ovsdbConfigurationService
 839                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 840             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 841             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 842             if (dpids == null || dpids.size() == 0) return 0L;
 843             return HexEncode.stringToLong((String) dpids.toArray()[0]);
 844         } catch (Exception e) {
 845             logger.error("Error finding Bridge's OF DPID", e);
 846             return 0L;
 847         }
 848     }
 849
 850     private String getInternalBridgeUUID (Node node, String bridgeName) {
 851         Preconditions.checkNotNull(ovsdbConfigurationService);
 852         try {
 853             Map<String, Row> bridgeTable =
 854                     ovsdbConfigurationService.getRows(node,
 855                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 856             if (bridgeTable == null) return null;
 857             for (String key : bridgeTable.keySet()) {
 858                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 859                 if (bridge.getName().equals(bridgeName)) return key;
 860             }
 861         } catch (Exception e) {
 862             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 863         }
 864         return null;
 865     }
 866 }
 867
 868