openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.MdsalUtils;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  26 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  27 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  28
  29 import com.google.common.base.Preconditions;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32
  33 import java.net.InetAddress;
  34 import java.net.UnknownHostException;
  35 import java.util.HashMap;
  36 import java.util.HashSet;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40
  41 /**
  42  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  43  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  44  * as well as the multi-tenant router forwarding provider.
  45  */
  46 public class NeutronL3Adapter {
  47
  48     /**
  49      * Logger instance.
  50      */
  51     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  52
  53     // The implementation for each of these services is resolved by the OSGi Service Manager
  54     private volatile ConfigurationService configurationService;
  55     private volatile TenantNetworkManager tenantNetworkManager;
  56     private volatile NodeCacheManager nodeCacheManager;
  57     private volatile INeutronNetworkCRUD neutronNetworkCache;
  58     private volatile INeutronSubnetCRUD neutronSubnetCache;
  59     private volatile INeutronPortCRUD neutronPortCache;
  60     private volatile L3ForwardingProvider l3ForwardingProvider;
  61     private volatile InboundNatProvider inboundNatProvider;
  62     private volatile OutboundNatProvider outboundNatProvider;
  63     private volatile ArpProvider arpProvider;
  64     private volatile RoutingProvider routingProvider;
  65
  66     private Set<String> inboundIpRewriteCache;
  67     private Set<String> outboundIpRewriteCache;
  68     private Set<String> inboundIpRewriteExclusionCache;
  69     private Set<String> outboundIpRewriteExclusionCache;
  70     private Set<String> routerInterfacesCache;
  71     private Set<String> staticArpEntryCache;
  72     private Set<String> l3ForwardingCache;
  73     private Set<String> defaultRouteCache;
  74     private Map<String, String> networkIdToRouterMacCache;
  75     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  76     private Boolean enabled = false;
  77
  78     void init() {
  79         logger.info(">>>>>> init {}", this.getClass());
  80         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  81         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  82             this.inboundIpRewriteCache = new HashSet<>();
  83             this.outboundIpRewriteCache = new HashSet<>();
  84             this.inboundIpRewriteExclusionCache = new HashSet<>();
  85             this.outboundIpRewriteExclusionCache = new HashSet<>();
  86             this.routerInterfacesCache = new HashSet<>();
  87             this.staticArpEntryCache = new HashSet<>();
  88             this.l3ForwardingCache = new HashSet<>();
  89             this.defaultRouteCache = new HashSet<>();
  90             this.networkIdToRouterMacCache = new HashMap<>();
  91             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  92
  93             this.enabled = true;
  94             logger.info("OVSDB L3 forwarding is enabled");
  95         } else {
  96             logger.debug("OVSDB L3 forwarding is disabled");
  97         }
  98     }
  99
 100     //
 101     // Callbacks from OVSDB's northbound handlers
 102     //
 103
 104     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 105         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 106         if (!this.enabled)
 107             return;
 108     }
 109
 110     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 111         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 112         if (!this.enabled)
 113             return;
 114
 115         final boolean isDelete = action == Action.DELETE;
 116
 117         // Treat the port event as a router interface event if the port belongs to router. This is a
 118         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 119         //
 120         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 121             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 122                 NeutronRouter_Interface neutronRouterInterface =
 123                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 124                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 125                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 126
 127                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 128             }
 129         } else {
 130             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 131             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 132             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 133             // there.
 134             //
 135             if (!isDelete) {
 136                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 137                     NeutronRouter_Interface neutronRouterInterface =
 138                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 139                     if (neutronRouterInterface != null) {
 140                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 141                     }
 142                 }
 143             }
 144             this.updateL3ForNeutronPort(neutronPort, isDelete);
 145         }
 146     }
 147
 148     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 149         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 150         if (!this.enabled)
 151             return;
 152     }
 153
 154     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 155                                                   final NeutronRouter_Interface neutronRouterInterface,
 156                                                   Action action) {
 157         logger.debug("Router interface {} got event {}. Subnet {}",
 158                      neutronRouterInterface.getPortUUID(),
 159                      action,
 160                      neutronRouterInterface.getSubnetUUID());
 161         if (!this.enabled)
 162             return;
 163
 164         final boolean isDelete = action == Action.DELETE;
 165
 166         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 167
 168         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 169         // see if they are affected by l3
 170         //
 171         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 172             boolean currPortShouldBeDeleted = false;
 173             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 174             if (isDelete) {
 175                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 176                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 177                         currPortShouldBeDeleted = true;
 178                         break;
 179                     }
 180                 }
 181             }
 182             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 183         }
 184     }
 185
 186     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 187                                              Action action) {
 188         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 189                      neutronFloatingIP.getFixedIPAddress(),
 190                      neutronFloatingIP.getFloatingIPAddress(),
 191                      neutronFloatingIP.getFloatingNetworkUUID());
 192         if (!this.enabled)
 193             return;
 194
 195         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 196     }
 197
 198     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 199         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 200         if (!this.enabled)
 201             return;
 202     }
 203
 204     //
 205     // Callbacks from OVSDB's southbound handler
 206     //
 207     public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
 208                                      final NeutronNetwork neutronNetwork, Action action) {
 209         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 210                      action, node, intf.getName(), neutronNetwork);
 211         if (!this.enabled)
 212             return;
 213
 214         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 215         if (neutronPort != null) {
 216             this.handleNeutronPortEvent(neutronPort, action);
 217         }
 218     }
 219
 220     //
 221     // Internal helpers
 222     //
 223     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 224
 225         final String networkUUID = neutronPort.getNetworkUUID();
 226         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 227
 228         // If there is no router interface handling the networkUUID, we are done
 229         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 230             return;
 231         }
 232
 233         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 234         // router interface are handled via handleNeutronRouterInterfaceEvent.
 235         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 236             return;
 237         }
 238
 239         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 240         final String providerSegmentationId = neutronNetwork != null ?
 241                                               neutronNetwork.getProviderSegmentationID() : null;
 242         final String tenantMac = neutronPort.getMacAddress();
 243
 244         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 245             tenantMac == null || tenantMac.isEmpty()) {
 246             return;  // done: go no further w/out all the info needed...
 247         }
 248
 249         final Action action = isDelete ? Action.DELETE : Action.ADD;
 250         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 251         if (nodes.isEmpty()) {
 252             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 253         }
 254         for (Node node : nodes) {
 255             final Long dpid = getDpidForIntegrationBridge(node);
 256             if (dpid == null) {
 257                 continue;
 258             }
 259
 260             final boolean tenantNetworkPresentInNode =
 261                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 262             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 263                 final String tenantIpStr = neutronIP.getIpAddress();
 264                 if (tenantIpStr.isEmpty()) {
 265                     continue;
 266                 }
 267
 268                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 269                 // still needed when routing to subnets non-local to node (bug 2076).
 270                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 271
 272                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 273                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 274                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 275             }
 276         }
 277     }
 278
 279     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 280                                            String macAddress, String ipStr,
 281                                            Action actionForNode) {
 282         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 283         // will look at desired action for node.
 284         //
 285         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 286         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 287
 288         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 289             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 290                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 291             return;
 292         }
 293         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 294             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 295                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 296             return;
 297         }
 298
 299         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 300                                                        macAddress, ipStr, actionForNode);
 301         if (status.isSuccess()) {
 302             // Update cache
 303             if (actionForNode == Action.ADD) {
 304                 l3ForwardingCache.add(cacheKey);
 305             } else {
 306                 l3ForwardingCache.remove(cacheKey);
 307             }
 308         }
 309     }
 310
 311     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 312                                              String macAddress,
 313                                              String address,
 314                                              Action actionForNode) {
 315         Status status;
 316         try {
 317             InetAddress inetAddress = InetAddress.getByName(address);
 318             status = l3ForwardingProvider == null ?
 319                      new Status(StatusCode.SUCCESS) :
 320                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 321                                                                       inetAddress, macAddress, actionForNode);
 322         } catch (UnknownHostException e) {
 323             status = new Status(StatusCode.BADREQUEST);
 324         }
 325
 326         if (status.isSuccess()) {
 327             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 328                          l3ForwardingProvider == null ? "skipped" : "programmed",
 329                          macAddress, address, node, actionForNode);
 330         } else {
 331             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 332                          macAddress, address, node, actionForNode, status);
 333         }
 334         return status;
 335     }
 336
 337     // --
 338
 339     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 340                                                        Boolean isDelete) {
 341         Preconditions.checkNotNull(destNeutronRouterInterface);
 342
 343         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 344         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 345         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 346         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 347         final NeutronNetwork neutronNetwork = subnet != null ?
 348                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 349         final String destinationSegmentationId = neutronNetwork != null ?
 350                                                  neutronNetwork.getProviderSegmentationID() : null;
 351         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 352         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 353         final String cidr = subnet != null ? subnet.getCidr() : null;
 354         final int mask = getMaskLenFromCidr(cidr);
 355
 356         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 357                      destNeutronRouterInterface, isDelete);
 358
 359         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 360             cidr == null || cidr.isEmpty() ||
 361             macAddress == null || macAddress.isEmpty() ||
 362             ipList == null || ipList.isEmpty()) {
 363             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 364                          destinationSegmentationId, cidr, macAddress, ipList);
 365             return;  // done: go no further w/out all the info needed...
 366         }
 367
 368         final Action action = isDelete ? Action.DELETE : Action.ADD;
 369
 370         // Keep cache for finding router's mac from network uuid -- add
 371         //
 372         if (! isDelete) {
 373             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 374             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 375         }
 376
 377         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 378         if (nodes.isEmpty()) {
 379             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 380         }
 381         for (Node node : nodes) {
 382             final Long dpid = getDpidForIntegrationBridge(node);
 383             if (dpid == null) {
 384                 continue;
 385             }
 386
 387             final Action actionForNode =
 388                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 389                     action : Action.DELETE;
 390
 391             for (Neutron_IPs neutronIP : ipList) {
 392                 final String ipStr = neutronIP.getIpAddress();
 393                 if (ipStr.isEmpty()) {
 394                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 395                             node.getNodeId().getValue(), ipStr);
 396                     continue;
 397                 }
 398
 399                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 400                 //
 401                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 402                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 403                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 404                                                               neutronNetwork, destinationSegmentationId,
 405                                                               macAddress, ipStr, mask, actionForNode,
 406                                                               true /*isReflexsive*/);
 407                 }
 408
 409                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 410             }
 411
 412             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 413             // for the external neutron networks.
 414             //
 415             {
 416                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 417                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 418                                                 cidr, actionForRewriteExclusion);
 419                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 420                                                 cidr, actionForRewriteExclusion);
 421             }
 422
 423             // Default route. For non-external subnet, make sure that there is none configured.
 424             //
 425             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 426                 final Action actionForNodeDefaultRoute =
 427                         isExternal ? actionForNode : Action.DELETE;
 428                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 429                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 430                                           actionForNodeDefaultRoute);
 431             }
 432         }
 433
 434         // Keep cache for finding router's mac from network uuid -- remove
 435         //
 436         if (isDelete) {
 437             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 438             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 439         }
 440     }
 441
 442     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 443                                                            final Long dpid,
 444                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 445                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 446                                                            final NeutronNetwork dstNeutronNetwork,
 447                                                            final String destinationSegmentationId,
 448                                                            final String dstMacAddress,
 449                                                            final String destIpStr,
 450                                                            final int destMask,
 451                                                            final Action actionForNode,
 452                                                            Boolean isReflexsive) {
 453         Preconditions.checkNotNull(srcNeutronRouterInterface);
 454         Preconditions.checkNotNull(dstNeutronRouterInterface);
 455
 456         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 457         if (sourceSubnetId == null) {
 458             logger.error("Could not get provider Subnet ID from router interface {}",
 459                          srcNeutronRouterInterface.getID());
 460             return;
 461         }
 462
 463         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 464         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 465         if (sourceNetworkId == null) {
 466             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 467             return;
 468         }
 469
 470         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 471         if (sourceNetwork == null) {
 472             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 473             return;
 474         }
 475
 476         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 477             // Isolate subnets from different tenants within the same router
 478             return;
 479         }
 480         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 481         if (sourceSegmentationId == null) {
 482             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 483             return;
 484         }
 485         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 486             // Skip 'self'
 487             return;
 488         }
 489
 490         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 491                                      dstMacAddress, destIpStr, destMask, actionForNode);
 492
 493         // Flip roles src->dst; dst->src
 494         if (isReflexsive) {
 495             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 496             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 497             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 498             final String cidr2 = sourceSubnet.getCidr();
 499             final int mask2 = getMaskLenFromCidr(cidr2);
 500
 501             if (cidr2 == null || cidr2.isEmpty() ||
 502                 macAddress2 == null || macAddress2.isEmpty() ||
 503                 ipList2 == null || ipList2.isEmpty()) {
 504                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 505                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 506                 return;  // done: go no further w/out all the info needed...
 507             }
 508
 509             for (Neutron_IPs neutronIP2 : ipList2) {
 510                 final String ipStr2 = neutronIP2.getIpAddress();
 511                 if (ipStr2.isEmpty()) {
 512                     continue;
 513                 }
 514                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 515                                                           srcNeutronRouterInterface,
 516                                                           sourceNetwork, sourceSegmentationId,
 517                                                           macAddress2, ipStr2, mask2, actionForNode,
 518                                                           false /*isReflexsive*/);
 519             }
 520         }
 521     }
 522
 523     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 524                                               String destinationSegmentationId,
 525                                               String macAddress, String ipStr, int mask,
 526                                               Action actionForNode) {
 527         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 528         // will look at desired action for node.
 529         //
 530         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 531                                 ipStr + "/" + Integer.toString(mask);
 532         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 533
 534         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 535             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 536                          "action {} is already done",
 537                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 538                          ipStr, mask, actionForNode);
 539             return;
 540         }
 541         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 542             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 543                          "action {} is already done",
 544                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 545                          ipStr, mask, actionForNode);
 546             return;
 547         }
 548
 549         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 550                                                           macAddress, ipStr, mask, actionForNode);
 551         if (status.isSuccess()) {
 552             // Update cache
 553             if (actionForNode == Action.ADD) {
 554                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 555                 routerInterfacesCache.add(cacheKey);
 556             } else {
 557                 // TODO: multiTenantAwareRouter.removeInterface(...);
 558                 routerInterfacesCache.remove(cacheKey);
 559             }
 560         }
 561     }
 562
 563     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 564                                                 String destinationSegmentationId,
 565                                                 String macAddress,
 566                                                 String address, int mask,
 567                                                 Action actionForNode) {
 568         Status status;
 569         try {
 570             InetAddress inetAddress = InetAddress.getByName(address);
 571             status = routingProvider == null ?
 572                      new Status(StatusCode.SUCCESS) :
 573                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 574                                                             macAddress, inetAddress, mask, actionForNode);
 575         } catch (UnknownHostException e) {
 576             status = new Status(StatusCode.BADREQUEST);
 577         }
 578
 579         if (status.isSuccess()) {
 580             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 581                          routingProvider == null ? "skipped" : "programmed",
 582                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 583                          actionForNode);
 584         } else {
 585             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 586                          macAddress, address, mask, node, sourceSegmentationId, destinationSegmentationId,
 587                          actionForNode, status);
 588         }
 589         return status;
 590     }
 591
 592     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 593                                         String macAddress, String ipStr,
 594                                         Action actionForNode) {
 595         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 596         // will look at desired action for node.
 597         //
 598         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 599         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 600
 601         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 602             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 603                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 604             return;
 605         }
 606         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 607             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 608                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 609             return;
 610         }
 611
 612         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 613                                                     macAddress, ipStr, actionForNode);
 614         if (status.isSuccess()) {
 615             // Update cache
 616             if (actionForNode == Action.ADD) {
 617                 staticArpEntryCache.add(cacheKey);
 618             } else {
 619                 staticArpEntryCache.remove(cacheKey);
 620             }
 621         }
 622     }
 623
 624     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 625                                                 String macAddress,
 626                                                 String address,
 627                                                 Action actionForNode) {
 628         Status status;
 629         try {
 630             InetAddress inetAddress = InetAddress.getByName(address);
 631             status = arpProvider == null ?
 632                      new Status(StatusCode.SUCCESS) :
 633                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 634                                                        macAddress, inetAddress, actionForNode);
 635         } catch (UnknownHostException e) {
 636             status = new Status(StatusCode.BADREQUEST);
 637         }
 638
 639         if (status.isSuccess()) {
 640             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 641                          arpProvider == null ? "skipped" : "programmed",
 642                          macAddress, address, node, actionForNode);
 643         } else {
 644             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 645                          macAddress, address, node, actionForNode, status);
 646         }
 647         return status;
 648     }
 649
 650     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 651                                                  final boolean isInbound, String cidr,
 652                                                  Action actionForRewriteExclusion) {
 653         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 654         // will look at desired action for node.
 655         //
 656         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 657         final Boolean isProgrammed = isInbound ?
 658                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 659                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 660
 661         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 662             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 663                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 664                          actionForRewriteExclusion);
 665             return;
 666         }
 667         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 668             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 669                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 670                          actionForRewriteExclusion);
 671             return;
 672         }
 673
 674         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 675                                                              isInbound, actionForRewriteExclusion);
 676         if (status.isSuccess()) {
 677             // Update cache
 678             if (actionForRewriteExclusion == Action.ADD) {
 679                 if (isInbound) {
 680                     inboundIpRewriteExclusionCache.add(cacheKey);
 681                 } else {
 682                     outboundIpRewriteExclusionCache.add(cacheKey);
 683                 }
 684             } else {
 685                 if (isInbound) {
 686                     inboundIpRewriteExclusionCache.remove(cacheKey);
 687                 } else {
 688                     outboundIpRewriteExclusionCache.remove(cacheKey);
 689                 }
 690             }
 691         }
 692     }
 693
 694     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 695                                                    final boolean isInbound, Action actionForNode) {
 696         Status status;
 697         if (isInbound) {
 698             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 699                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 700                                                                   actionForNode);
 701         } else {
 702             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 703                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 704                                                                    actionForNode);
 705         }
 706
 707         if (status.isSuccess()) {
 708             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 709             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 710                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 711                          cidr, node, actionForNode);
 712         } else {
 713             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 714                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 715         }
 716         return status;
 717     }
 718
 719     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 720                                            String defaultGatewayMacAddress, String gatewayIp,
 721                                            Action actionForNodeDefaultRoute) {
 722         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 723         // will look at desired action for node.
 724         //
 725         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 726         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 727
 728         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 729             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 730                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 731                          actionForNodeDefaultRoute);
 732             return;
 733         }
 734         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 735             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 736                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 737                          actionForNodeDefaultRoute);
 738             return;
 739         }
 740
 741         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 742                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 743         if (status.isSuccess()) {
 744             // Update cache
 745             if (actionForNodeDefaultRoute == Action.ADD) {
 746                 defaultRouteCache.add(cacheKey);
 747             } else {
 748                 defaultRouteCache.remove(cacheKey);
 749             }
 750         }
 751     }
 752
 753     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 754                                           String defaultGatewayMacAddress,
 755                                           String gatewayIp,
 756                                           Action actionForNodeDefaultRoute) {
 757         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 758         if (defaultGatewayMacAddress == null) {
 759             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 760                          gatewayIp, node, actionForNodeDefaultRoute);
 761             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 762         }
 763
 764         Status status;
 765         try {
 766             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 767             status = routingProvider == null ?
 768                      new Status(StatusCode.SUCCESS) :
 769                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 770                                                               defaultGatewayMacAddress, inetAddress,
 771                                                               actionForNodeDefaultRoute);
 772         } catch (UnknownHostException e) {
 773             status = new Status(StatusCode.BADREQUEST);
 774         }
 775
 776         if (status.isSuccess()) {
 777             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 778                          routingProvider == null ? "skipped" : "programmed",
 779                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 780         } else {
 781             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 782                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 783         }
 784         return status;
 785     }
 786
 787     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 788         Preconditions.checkNotNull(neutronFloatingIP);
 789
 790         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 791         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 792
 793         // If there is no router interface handling the networkUUID, we are done
 794         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 795             return;
 796         }
 797
 798         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 799         final String providerSegmentationId = neutronNetwork != null ?
 800                                               neutronNetwork.getProviderSegmentationID() : null;
 801         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 802         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 803
 804         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 805             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 806             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 807             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 808             return;  // done: go no further w/out all the info needed...
 809         }
 810
 811         final Action action = isDelete ? Action.DELETE : Action.ADD;
 812         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 813         if (nodes.isEmpty()) {
 814             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 815         }
 816         for (Node node : nodes) {
 817             final Long dpid = getDpidForIntegrationBridge(node);
 818             if (dpid == null) {
 819                 continue;
 820             }
 821
 822             final Action actionForNode =
 823                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 824                     action : Action.DELETE;
 825
 826             // Rewrite from float to fixed and vice-versa
 827             //
 828             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 829                                    floatingIpAddress, fixedIPAddress, actionForNode);
 830             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 831                                    fixedIPAddress, floatingIpAddress, actionForNode);
 832
 833             // Respond to arps for the floating ip address
 834             //
 835             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 836                                    actionForNode);
 837         }
 838     }
 839
 840     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 841                                         final boolean isInbound,
 842                                         String matchAddress, String rewriteAddress,
 843                                         Action actionForNode) {
 844         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 845         // will look at desired action for node.
 846         //
 847         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 848                                 matchAddress + ":" + rewriteAddress;
 849         final Boolean isProgrammed = isInbound ?
 850                                      inboundIpRewriteCache.contains(cacheKey) :
 851                                      outboundIpRewriteCache.contains(cacheKey);
 852
 853         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 854             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 855                          " is already done",
 856                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 857                          matchAddress, rewriteAddress, actionForNode);
 858             return;
 859         }
 860         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 861             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 862                          " is already done",
 863                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 864                          matchAddress, rewriteAddress, actionForNode);
 865             return;
 866         }
 867
 868         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 869                                                     matchAddress, rewriteAddress, actionForNode);
 870         if (status.isSuccess()) {
 871             // Update cache
 872             if (actionForNode == Action.ADD) {
 873                 if (isInbound) {
 874                     inboundIpRewriteCache.add(cacheKey);
 875                 } else {
 876                     outboundIpRewriteCache.add(cacheKey);
 877                 }
 878             } else {
 879                 if (isInbound) {
 880                     inboundIpRewriteCache.remove(cacheKey);
 881                 } else {
 882                     outboundIpRewriteCache.remove(cacheKey);
 883                 }
 884             }
 885         }
 886     }
 887
 888     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 889                                           final boolean isInbound,
 890                                           String matchAddress, String rewriteAddress,
 891                                           Action actionForNode) {
 892         Status status;
 893         try {
 894             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 895             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 896             if (isInbound) {
 897                 status = inboundNatProvider == null ?
 898                          new Status(StatusCode.SUCCESS) :
 899                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 900                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 901             } else {
 902                 status = outboundNatProvider == null ?
 903                          new Status(StatusCode.SUCCESS) :
 904                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 905                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 906             }
 907         } catch (UnknownHostException e) {
 908             status = new Status(StatusCode.BADREQUEST);
 909         }
 910
 911         if (status.isSuccess()) {
 912             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 913             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 914                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 915                          matchAddress, rewriteAddress, node, actionForNode);
 916         } else {
 917             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 918                          (isInbound ? "inbound" : "outbound"),
 919                          matchAddress, rewriteAddress, node, actionForNode, status);
 920         }
 921         return status;
 922     }
 923
 924     private int getMaskLenFromCidr(String cidr) {
 925         if (cidr == null) return 0;
 926         String[] splits = cidr.split("/");
 927         if (splits.length != 2) return 0;
 928
 929         int result;
 930         try {
 931             result = Integer.parseInt(splits[1].trim());
 932         }
 933         catch (NumberFormatException nfe)
 934         {
 935             result = 0;
 936         }
 937         return result;
 938     }
 939
 940     private Long getDpidForIntegrationBridge(Node node) {
 941         // Check if node is integration bridge; and only then return its dpid
 942         if (MdsalUtils.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
 943             return MdsalUtils.getDataPathId(node);
 944         }
 945         return null;
 946     }
 947 }