openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import java.util.ArrayList;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  27 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  28
  29 import com.google.common.base.Preconditions;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32
  33 import java.net.InetAddress;
  34 import java.net.UnknownHostException;
  35 import java.util.HashMap;
  36 import java.util.HashSet;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40
  41 /**
  42  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  43  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  44  * as well as the multi-tenant router forwarding provider.
  45  */
  46 public class NeutronL3Adapter {
  47
  48     /**
  49      * Logger instance.
  50      */
  51     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  52
  53     // The implementation for each of these services is resolved by the OSGi Service Manager
  54     private volatile ConfigurationService configurationService;
  55     private volatile TenantNetworkManager tenantNetworkManager;
  56     private volatile OvsdbConnectionService connectionService;
  57     private volatile INeutronNetworkCRUD neutronNetworkCache;
  58     private volatile INeutronSubnetCRUD neutronSubnetCache;
  59     private volatile INeutronPortCRUD neutronPortCache;
  60     private volatile L3ForwardingProvider l3ForwardingProvider;
  61     private volatile InboundNatProvider inboundNatProvider;
  62     private volatile OutboundNatProvider outboundNatProvider;
  63     private volatile ArpProvider arpProvider;
  64     private volatile RoutingProvider routingProvider;
  65
  66     private Set<String> inboundIpRewriteCache;
  67     private Set<String> outboundIpRewriteCache;
  68     private Set<String> inboundIpRewriteExclusionCache;
  69     private Set<String> outboundIpRewriteExclusionCache;
  70     private Set<String> routerInterfacesCache;
  71     private Set<String> staticArpEntryCache;
  72     private Set<String> l3ForwardingCache;
  73     private Set<String> defaultRouteCache;
  74     private Map<String, String> networkIdToRouterMacCache;
  75     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  76     private Boolean enabled = false;
  77
  78     void init() {
  79         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  80         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  81             this.inboundIpRewriteCache = new HashSet<>();
  82             this.outboundIpRewriteCache = new HashSet<>();
  83             this.inboundIpRewriteExclusionCache = new HashSet<>();
  84             this.outboundIpRewriteExclusionCache = new HashSet<>();
  85             this.routerInterfacesCache = new HashSet<>();
  86             this.staticArpEntryCache = new HashSet<>();
  87             this.l3ForwardingCache = new HashSet<>();
  88             this.defaultRouteCache = new HashSet<>();
  89             this.networkIdToRouterMacCache = new HashMap<>();
  90             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  91
  92             this.enabled = true;
  93             logger.info("OVSDB L3 forwarding is enabled");
  94         } else {
  95             logger.debug("OVSDB L3 forwarding is disabled");
  96         }
  97     }
  98
  99     //
 100     // Callbacks from OVSDB's northbound handlers
 101     //
 102
 103     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 104         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 105         if (!this.enabled)
 106             return;
 107     }
 108
 109     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 110         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 111         if (!this.enabled)
 112             return;
 113
 114         final boolean isDelete = action == Action.DELETE;
 115
 116         // Treat the port event as a router interface event if the port belongs to router. This is a
 117         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 118         //
 119         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 120             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 121                 NeutronRouter_Interface neutronRouterInterface =
 122                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 123                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 124                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 125
 126                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 127             }
 128         } else {
 129             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 130             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 131             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 132             // there.
 133             //
 134             if (!isDelete) {
 135                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 136                     NeutronRouter_Interface neutronRouterInterface =
 137                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 138                     if (neutronRouterInterface != null) {
 139                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 140                     }
 141                 }
 142             }
 143             this.updateL3ForNeutronPort(neutronPort, isDelete);
 144         }
 145     }
 146
 147     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 148         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 149         if (!this.enabled)
 150             return;
 151     }
 152
 153     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 154                                                   final NeutronRouter_Interface neutronRouterInterface,
 155                                                   Action action) {
 156         logger.debug("Router interface {} got event {}. Subnet {}",
 157                      neutronRouterInterface.getPortUUID(),
 158                      action,
 159                      neutronRouterInterface.getSubnetUUID());
 160         if (!this.enabled)
 161             return;
 162
 163         final boolean isDelete = action == Action.DELETE;
 164
 165         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 166
 167         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 168         // see if they are affected by l3
 169         //
 170         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 171             boolean currPortShouldBeDeleted = false;
 172             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 173             if (isDelete) {
 174                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 175                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 176                         currPortShouldBeDeleted = true;
 177                         break;
 178                     }
 179                 }
 180             }
 181             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 182         }
 183     }
 184
 185     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 186                                              Action action) {
 187         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 188                      neutronFloatingIP.getFixedIPAddress(),
 189                      neutronFloatingIP.getFloatingIPAddress(),
 190                      neutronFloatingIP.getFloatingNetworkUUID());
 191         if (!this.enabled)
 192             return;
 193
 194         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 195     }
 196
 197     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 198         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 199         if (!this.enabled)
 200             return;
 201     }
 202
 203     //
 204     // Callbacks from OVSDB's southbound handler
 205     //
 206     public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
 207                                      final NeutronNetwork neutronNetwork, Action action) {
 208         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 209                      action, node, intf.getName(), neutronNetwork);
 210         if (!this.enabled)
 211             return;
 212
 213         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 214         if (neutronPort != null) {
 215             this.handleNeutronPortEvent(neutronPort, action);
 216         }
 217     }
 218
 219     //
 220     // Internal helpers
 221     //
 222     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 223
 224         final String networkUUID = neutronPort.getNetworkUUID();
 225         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 226
 227         // If there is no router interface handling the networkUUID, we are done
 228         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 229             return;
 230         }
 231
 232         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 233         // router interface are handled via handleNeutronRouterInterfaceEvent.
 234         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 235             return;
 236         }
 237
 238         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 239         final String providerSegmentationId = neutronNetwork != null ?
 240                                               neutronNetwork.getProviderSegmentationID() : null;
 241         final String tenantMac = neutronPort.getMacAddress();
 242
 243         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 244             tenantMac == null || tenantMac.isEmpty()) {
 245             return;  // done: go no further w/out all the info needed...
 246         }
 247
 248         final Action action = isDelete ? Action.DELETE : Action.ADD;
 249         List<Node> nodes = connectionService.getBridgeNodes();
 250         if (nodes.isEmpty()) {
 251             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 252         }
 253         for (Node node : nodes) {
 254             final Long dpid = getDpid(node);
 255             final boolean tenantNetworkPresentInNode =
 256                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 257             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 258                 final String tenantIpStr = neutronIP.getIpAddress();
 259                 if (tenantIpStr.isEmpty()) {
 260                     continue;
 261                 }
 262
 263                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 264                 // still needed when routing to subnets non-local to node (bug 2076).
 265                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 266
 267                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 268                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 269                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 270             }
 271         }
 272     }
 273
 274     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 275                                            String macAddress, String ipStr,
 276                                            Action actionForNode) {
 277         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 278         // will look at desired action for node.
 279         //
 280         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 281         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 282
 283         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 284             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 285                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 286             return;
 287         }
 288         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 289             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 290                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 291             return;
 292         }
 293
 294         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 295                                                        macAddress, ipStr, actionForNode);
 296         if (status.isSuccess()) {
 297             // Update cache
 298             if (actionForNode == Action.ADD) {
 299                 l3ForwardingCache.add(cacheKey);
 300             } else {
 301                 l3ForwardingCache.remove(cacheKey);
 302             }
 303         }
 304     }
 305
 306     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 307                                              String macAddress,
 308                                              String address,
 309                                              Action actionForNode) {
 310         Status status;
 311         try {
 312             InetAddress inetAddress = InetAddress.getByName(address);
 313             status = l3ForwardingProvider == null ?
 314                      new Status(StatusCode.SUCCESS) :
 315                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 316                                                                       inetAddress, macAddress, actionForNode);
 317         } catch (UnknownHostException e) {
 318             status = new Status(StatusCode.BADREQUEST);
 319         }
 320
 321         if (status.isSuccess()) {
 322             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 323                          l3ForwardingProvider == null ? "skipped" : "programmed",
 324                          macAddress, address, node, actionForNode);
 325         } else {
 326             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 327                          macAddress, address, node, actionForNode, status);
 328         }
 329         return status;
 330     }
 331
 332     // --
 333
 334     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 335                                                        Boolean isDelete) {
 336         Preconditions.checkNotNull(destNeutronRouterInterface);
 337
 338         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 339         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 340         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 341         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 342         final NeutronNetwork neutronNetwork = subnet != null ?
 343                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 344         final String destinationSegmentationId = neutronNetwork != null ?
 345                                                  neutronNetwork.getProviderSegmentationID() : null;
 346         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 347         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 348         final String cidr = subnet != null ? subnet.getCidr() : null;
 349         final int mask = getMaskLenFromCidr(cidr);
 350
 351         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 352                      destNeutronRouterInterface, isDelete);
 353
 354         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 355             cidr == null || cidr.isEmpty() ||
 356             macAddress == null || macAddress.isEmpty() ||
 357             ipList == null || ipList.isEmpty()) {
 358             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 359                          destinationSegmentationId, cidr, macAddress, ipList);
 360             return;  // done: go no further w/out all the info needed...
 361         }
 362
 363         final Action action = isDelete ? Action.DELETE : Action.ADD;
 364
 365         // Keep cache for finding router's mac from network uuid -- add
 366         //
 367         if (! isDelete) {
 368             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 369             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 370         }
 371
 372         List<Node> nodes = connectionService.getBridgeNodes();
 373         if (nodes.isEmpty()) {
 374             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 375         }
 376         for (Node node : nodes) {
 377             final Long dpid = getDpid(node);
 378             final Action actionForNode =
 379                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 380                     action : Action.DELETE;
 381
 382             for (Neutron_IPs neutronIP : ipList) {
 383                 final String ipStr = neutronIP.getIpAddress();
 384                 if (ipStr.isEmpty()) {
 385                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 386                             node.getNodeId().getValue(), ipStr);
 387                     continue;
 388                 }
 389
 390                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 391                 //
 392                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 393                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 394                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 395                                                               neutronNetwork, destinationSegmentationId,
 396                                                               macAddress, ipStr, mask, actionForNode,
 397                                                               true /*isReflexsive*/);
 398                 }
 399
 400                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 401             }
 402
 403             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 404             // for the external neutron networks.
 405             //
 406             {
 407                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 408                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 409                                                 cidr, actionForRewriteExclusion);
 410                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 411                                                 cidr, actionForRewriteExclusion);
 412             }
 413
 414             // Default route. For non-external subnet, make sure that there is none configured.
 415             //
 416             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 417                 final Action actionForNodeDefaultRoute =
 418                         isExternal ? actionForNode : Action.DELETE;
 419                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 420                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 421                                           actionForNodeDefaultRoute);
 422             }
 423         }
 424
 425         // Keep cache for finding router's mac from network uuid -- remove
 426         //
 427         if (isDelete) {
 428             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 429             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 430         }
 431     }
 432
 433     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 434                                                            final Long dpid,
 435                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 436                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 437                                                            final NeutronNetwork dstNeutronNetwork,
 438                                                            final String destinationSegmentationId,
 439                                                            final String dstMacAddress,
 440                                                            final String destIpStr,
 441                                                            final int destMask,
 442                                                            final Action actionForNode,
 443                                                            Boolean isReflexsive) {
 444         Preconditions.checkNotNull(srcNeutronRouterInterface);
 445         Preconditions.checkNotNull(dstNeutronRouterInterface);
 446
 447         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 448         if (sourceSubnetId == null) {
 449             logger.error("Could not get provider Subnet ID from router interface {}",
 450                          srcNeutronRouterInterface.getID());
 451             return;
 452         }
 453
 454         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 455         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 456         if (sourceNetworkId == null) {
 457             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 458             return;
 459         }
 460
 461         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 462         if (sourceNetwork == null) {
 463             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 464             return;
 465         }
 466
 467         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 468             // Isolate subnets from different tenants within the same router
 469             return;
 470         }
 471         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 472         if (sourceSegmentationId == null) {
 473             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 474             return;
 475         }
 476         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 477             // Skip 'self'
 478             return;
 479         }
 480
 481         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 482                                      dstMacAddress, destIpStr, destMask, actionForNode);
 483
 484         // Flip roles src->dst; dst->src
 485         if (isReflexsive) {
 486             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 487             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 488             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 489             final String cidr2 = sourceSubnet.getCidr();
 490             final int mask2 = getMaskLenFromCidr(cidr2);
 491
 492             if (cidr2 == null || cidr2.isEmpty() ||
 493                 macAddress2 == null || macAddress2.isEmpty() ||
 494                 ipList2 == null || ipList2.isEmpty()) {
 495                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 496                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 497                 return;  // done: go no further w/out all the info needed...
 498             }
 499
 500             for (Neutron_IPs neutronIP2 : ipList2) {
 501                 final String ipStr2 = neutronIP2.getIpAddress();
 502                 if (ipStr2.isEmpty()) {
 503                     continue;
 504                 }
 505                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 506                                                           srcNeutronRouterInterface,
 507                                                           sourceNetwork, sourceSegmentationId,
 508                                                           macAddress2, ipStr2, mask2, actionForNode,
 509                                                           false /*isReflexsive*/);
 510             }
 511         }
 512     }
 513
 514     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 515                                               String destinationSegmentationId,
 516                                               String macAddress, String ipStr, int mask,
 517                                               Action actionForNode) {
 518         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 519         // will look at desired action for node.
 520         //
 521         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 522                                 ipStr + "/" + Integer.toString(mask);
 523         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 524
 525         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 526             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 527                          "action {} is already done",
 528                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 529                          ipStr, mask, actionForNode);
 530             return;
 531         }
 532         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 533             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 534                          "action {} is already done",
 535                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 536                          ipStr, mask, actionForNode);
 537             return;
 538         }
 539
 540         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 541                                                           macAddress, ipStr, mask, actionForNode);
 542         if (status.isSuccess()) {
 543             // Update cache
 544             if (actionForNode == Action.ADD) {
 545                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 546                 routerInterfacesCache.add(cacheKey);
 547             } else {
 548                 // TODO: multiTenantAwareRouter.removeInterface(...);
 549                 routerInterfacesCache.remove(cacheKey);
 550             }
 551         }
 552     }
 553
 554     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 555                                                 String destinationSegmentationId,
 556                                                 String macAddress,
 557                                                 String address, int mask,
 558                                                 Action actionForNode) {
 559         Status status;
 560         try {
 561             InetAddress inetAddress = InetAddress.getByName(address);
 562             status = routingProvider == null ?
 563                      new Status(StatusCode.SUCCESS) :
 564                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 565                                                             macAddress, inetAddress, mask, actionForNode);
 566         } catch (UnknownHostException e) {
 567             status = new Status(StatusCode.BADREQUEST);
 568         }
 569
 570         if (status.isSuccess()) {
 571             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 572                          routingProvider == null ? "skipped" : "programmed",
 573                          macAddress, address, mask, node, actionForNode);
 574         } else {
 575             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 576                          macAddress, address, mask, node, actionForNode, status);
 577         }
 578         return status;
 579     }
 580
 581     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 582                                         String macAddress, String ipStr,
 583                                         Action actionForNode) {
 584         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 585         // will look at desired action for node.
 586         //
 587         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 588         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 589
 590         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 591             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 592                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 593             return;
 594         }
 595         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 596             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 597                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 598             return;
 599         }
 600
 601         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 602                                                     macAddress, ipStr, actionForNode);
 603         if (status.isSuccess()) {
 604             // Update cache
 605             if (actionForNode == Action.ADD) {
 606                 staticArpEntryCache.add(cacheKey);
 607             } else {
 608                 staticArpEntryCache.remove(cacheKey);
 609             }
 610         }
 611     }
 612
 613     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 614                                                 String macAddress,
 615                                                 String address,
 616                                                 Action actionForNode) {
 617         Status status;
 618         try {
 619             InetAddress inetAddress = InetAddress.getByName(address);
 620             status = arpProvider == null ?
 621                      new Status(StatusCode.SUCCESS) :
 622                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 623                                                        macAddress, inetAddress, actionForNode);
 624         } catch (UnknownHostException e) {
 625             status = new Status(StatusCode.BADREQUEST);
 626         }
 627
 628         if (status.isSuccess()) {
 629             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 630                          arpProvider == null ? "skipped" : "programmed",
 631                          macAddress, address, node, actionForNode);
 632         } else {
 633             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 634                          macAddress, address, node, actionForNode, status);
 635         }
 636         return status;
 637     }
 638
 639     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 640                                                  final boolean isInbound, String cidr,
 641                                                  Action actionForRewriteExclusion) {
 642         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 643         // will look at desired action for node.
 644         //
 645         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 646         final Boolean isProgrammed = isInbound ?
 647                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 648                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 649
 650         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 651             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 652                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 653                          actionForRewriteExclusion);
 654             return;
 655         }
 656         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 657             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 658                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 659                          actionForRewriteExclusion);
 660             return;
 661         }
 662
 663         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 664                                                              isInbound, actionForRewriteExclusion);
 665         if (status.isSuccess()) {
 666             // Update cache
 667             if (actionForRewriteExclusion == Action.ADD) {
 668                 if (isInbound) {
 669                     inboundIpRewriteExclusionCache.add(cacheKey);
 670                 } else {
 671                     outboundIpRewriteExclusionCache.add(cacheKey);
 672                 }
 673             } else {
 674                 if (isInbound) {
 675                     inboundIpRewriteExclusionCache.remove(cacheKey);
 676                 } else {
 677                     outboundIpRewriteExclusionCache.remove(cacheKey);
 678                 }
 679             }
 680         }
 681     }
 682
 683     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 684                                                    final boolean isInbound, Action actionForNode) {
 685         Status status;
 686         if (isInbound) {
 687             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 688                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 689                                                                   actionForNode);
 690         } else {
 691             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 692                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 693                                                                    actionForNode);
 694         }
 695
 696         if (status.isSuccess()) {
 697             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 698             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 699                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 700                          cidr, node, actionForNode);
 701         } else {
 702             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 703                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 704         }
 705         return status;
 706     }
 707
 708     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 709                                            String defaultGatewayMacAddress, String gatewayIp,
 710                                            Action actionForNodeDefaultRoute) {
 711         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 712         // will look at desired action for node.
 713         //
 714         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 715         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 716
 717         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 718             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 719                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 720                          actionForNodeDefaultRoute);
 721             return;
 722         }
 723         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 724             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 725                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 726                          actionForNodeDefaultRoute);
 727             return;
 728         }
 729
 730         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 731                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 732         if (status.isSuccess()) {
 733             // Update cache
 734             if (actionForNodeDefaultRoute == Action.ADD) {
 735                 defaultRouteCache.add(cacheKey);
 736             } else {
 737                 defaultRouteCache.remove(cacheKey);
 738             }
 739         }
 740     }
 741
 742     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 743                                           String defaultGatewayMacAddress,
 744                                           String gatewayIp,
 745                                           Action actionForNodeDefaultRoute) {
 746         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 747         if (defaultGatewayMacAddress == null) {
 748             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 749                          gatewayIp, node, actionForNodeDefaultRoute);
 750             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 751         }
 752
 753         Status status;
 754         try {
 755             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 756             status = routingProvider == null ?
 757                      new Status(StatusCode.SUCCESS) :
 758                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 759                                                               defaultGatewayMacAddress, inetAddress,
 760                                                               actionForNodeDefaultRoute);
 761         } catch (UnknownHostException e) {
 762             status = new Status(StatusCode.BADREQUEST);
 763         }
 764
 765         if (status.isSuccess()) {
 766             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 767                          routingProvider == null ? "skipped" : "programmed",
 768                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 769         } else {
 770             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 771                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 772         }
 773         return status;
 774     }
 775
 776     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 777         Preconditions.checkNotNull(neutronFloatingIP);
 778
 779         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 780         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 781
 782         // If there is no router interface handling the networkUUID, we are done
 783         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 784             return;
 785         }
 786
 787         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 788         final String providerSegmentationId = neutronNetwork != null ?
 789                                               neutronNetwork.getProviderSegmentationID() : null;
 790         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 791         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 792
 793         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 794             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 795             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 796             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 797             return;  // done: go no further w/out all the info needed...
 798         }
 799
 800         final Action action = isDelete ? Action.DELETE : Action.ADD;
 801         List<Node> nodes = connectionService.getBridgeNodes();
 802         if (nodes.isEmpty()) {
 803             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 804         }
 805         for (Node node : nodes) {
 806             final Long dpid = getDpid(node);
 807             final Action actionForNode =
 808                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 809                     action : Action.DELETE;
 810
 811             // Rewrite from float to fixed and vice-versa
 812             //
 813             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 814                                    floatingIpAddress, fixedIPAddress, actionForNode);
 815             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 816                                    fixedIPAddress, floatingIpAddress, actionForNode);
 817
 818             // Respond to arps for the floating ip address
 819             //
 820             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 821                                    actionForNode);
 822         }
 823     }
 824
 825     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 826                                         final boolean isInbound,
 827                                         String matchAddress, String rewriteAddress,
 828                                         Action actionForNode) {
 829         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 830         // will look at desired action for node.
 831         //
 832         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 833                                 matchAddress + ":" + rewriteAddress;
 834         final Boolean isProgrammed = isInbound ?
 835                                      inboundIpRewriteCache.contains(cacheKey) :
 836                                      outboundIpRewriteCache.contains(cacheKey);
 837
 838         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 839             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 840                          " is already done",
 841                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 842                          matchAddress, rewriteAddress, actionForNode);
 843             return;
 844         }
 845         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 846             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 847                          " is already done",
 848                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 849                          matchAddress, rewriteAddress, actionForNode);
 850             return;
 851         }
 852
 853         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 854                                                     matchAddress, rewriteAddress, actionForNode);
 855         if (status.isSuccess()) {
 856             // Update cache
 857             if (actionForNode == Action.ADD) {
 858                 if (isInbound) {
 859                     inboundIpRewriteCache.add(cacheKey);
 860                 } else {
 861                     outboundIpRewriteCache.add(cacheKey);
 862                 }
 863             } else {
 864                 if (isInbound) {
 865                     inboundIpRewriteCache.remove(cacheKey);
 866                 } else {
 867                     outboundIpRewriteCache.remove(cacheKey);
 868                 }
 869             }
 870         }
 871     }
 872
 873     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 874                                           final boolean isInbound,
 875                                           String matchAddress, String rewriteAddress,
 876                                           Action actionForNode) {
 877         Status status;
 878         try {
 879             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 880             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 881             if (isInbound) {
 882                 status = inboundNatProvider == null ?
 883                          new Status(StatusCode.SUCCESS) :
 884                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 885                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 886             } else {
 887                 status = outboundNatProvider == null ?
 888                          new Status(StatusCode.SUCCESS) :
 889                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 890                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 891             }
 892         } catch (UnknownHostException e) {
 893             status = new Status(StatusCode.BADREQUEST);
 894         }
 895
 896         if (status.isSuccess()) {
 897             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 898             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 899                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 900                          matchAddress, rewriteAddress, node, actionForNode);
 901         } else {
 902             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 903                          (isInbound ? "inbound" : "outbound"),
 904                          matchAddress, rewriteAddress, node, actionForNode, status);
 905         }
 906         return status;
 907     }
 908
 909     private int getMaskLenFromCidr(String cidr) {
 910         if (cidr == null) return 0;
 911         String[] splits = cidr.split("/");
 912         if (splits.length != 2) return 0;
 913
 914         int result;
 915         try {
 916             result = Integer.parseInt(splits[1].trim());
 917         }
 918         catch (NumberFormatException nfe)
 919         {
 920             result = 0;
 921         }
 922         return result;
 923     }
 924
 925     private Long getDpid(Node node) {
 926         /* TODO SB_MIGRATION */
 927         // may need to go from OvsdbNode to BridgeNode
 928         // get integration bridge on this node and then get dpid
 929         return MdsalUtils.getDataPathId(node);
 930     }
 931 }