openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.controller.networkconfig.neutron.INeutronNetworkCRUD;
  14 import org.opendaylight.controller.networkconfig.neutron.INeutronPortCRUD;
  15 import org.opendaylight.controller.networkconfig.neutron.INeutronSubnetCRUD;
  16 import org.opendaylight.controller.networkconfig.neutron.NeutronFloatingIP;
  17 import org.opendaylight.controller.networkconfig.neutron.NeutronNetwork;
  18 import org.opendaylight.controller.networkconfig.neutron.NeutronPort;
  19 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter;
  20 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter_Interface;
  21 import org.opendaylight.controller.networkconfig.neutron.NeutronSubnet;
  22 import org.opendaylight.controller.networkconfig.neutron.Neutron_IPs;
  23 import org.opendaylight.controller.sal.core.Node;
  24 import org.opendaylight.controller.sal.utils.HexEncode;
  25 import org.opendaylight.controller.sal.utils.Status;
  26 import org.opendaylight.controller.sal.utils.StatusCode;
  27 import org.opendaylight.ovsdb.lib.notation.Row;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.MultiTenantAwareRouter;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  33 import org.opendaylight.ovsdb.plugin.api.OvsdbConfigurationService;
  34 import org.opendaylight.ovsdb.plugin.api.OvsdbConnectionService;
  35 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  41 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  42 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  43
  44 import com.google.common.base.Preconditions;
  45 import org.slf4j.Logger;
  46 import org.slf4j.LoggerFactory;
  47
  48 import java.net.InetAddress;
  49 import java.net.UnknownHostException;
  50 import java.util.HashMap;
  51 import java.util.HashSet;
  52 import java.util.List;
  53 import java.util.Map;
  54 import java.util.Set;
  55
  56 /**
  57  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  58  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  59  * as well as the multi-tenant router forwarding provider.
  60  */
  61 public class NeutronL3Adapter {
  62
  63     /**
  64      * Logger instance.
  65      */
  66     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  67
  68     // The implementation for each of these services is resolved by the OSGi Service Manager
  69     private volatile ConfigurationService configurationService;
  70     private volatile TenantNetworkManager tenantNetworkManager;
  71     private volatile NetworkingProviderManager networkingProviderManager;
  72     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  73     private volatile OvsdbConnectionService connectionService;
  74     private volatile INeutronNetworkCRUD neutronNetworkCache;
  75     private volatile INeutronSubnetCRUD neutronSubnetCache;
  76     private volatile INeutronPortCRUD neutronPortCache;
  77     private volatile MultiTenantAwareRouter multiTenantAwareRouter;
  78     private volatile L3ForwardingProvider l3ForwardingProvider;
  79     private volatile InboundNatProvider inboundNatProvider;
  80     private volatile OutboundNatProvider outboundNatProvider;
  81     private volatile ArpProvider arpProvider;
  82     private volatile RoutingProvider routingProvider;
  83
  84     private Set<String> inboundIpRewriteCache;
  85     private Set<String> outboundIpRewriteCache;
  86     private Set<String> inboundIpRewriteExclusionCache;
  87     private Set<String> outboundIpRewriteExclusionCache;
  88     private Set<String> routerInterfacesCache;
  89     private Set<String> staticArpEntryCache;
  90     private Set<String> l3ForwardingCache;
  91     private Set<String> defaultRouteCache;
  92     private Map<String, String> networkIdToRouterMacCache;
  93     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  94     private Boolean enabled = false;
  95
  96     void init() {
  97         final String enabledPropertyStr = System.getProperty("ovsdb.l3.fwd.enabled");
  98         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  99             this.inboundIpRewriteCache = new HashSet<>();
 100             this.outboundIpRewriteCache = new HashSet<>();
 101             this.inboundIpRewriteExclusionCache = new HashSet<>();
 102             this.outboundIpRewriteExclusionCache = new HashSet<>();
 103             this.routerInterfacesCache = new HashSet<>();
 104             this.staticArpEntryCache = new HashSet<>();
 105             this.l3ForwardingCache = new HashSet<>();
 106             this.defaultRouteCache = new HashSet<>();
 107             this.networkIdToRouterMacCache = new HashMap<>();
 108             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 109
 110             this.enabled = true;
 111             logger.info("OVSDB L3 forwarding is enabled");
 112         } else {
 113             logger.debug("OVSDB L3 forwarding is disabled");
 114         }
 115     }
 116
 117     //
 118     // Callbacks from OVSDB's northbound handlers
 119     //
 120
 121     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 122         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 123         if (!this.enabled)
 124             return;
 125     }
 126
 127     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 128         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 129         if (!this.enabled)
 130             return;
 131
 132         final boolean isDelete = action == Action.DELETE;
 133
 134         // Treat the port event as a router interface event if the port belongs to router. This is a
 135         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 136         //
 137         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 138             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 139                 NeutronRouter_Interface neutronRouterInterface =
 140                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 141                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 142                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 143
 144                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 145             }
 146         } else {
 147             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 148             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 149             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 150             // there.
 151             //
 152             if (!isDelete) {
 153                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 154                     NeutronRouter_Interface neutronRouterInterface =
 155                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 156                     if (neutronRouterInterface != null) {
 157                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 158                     }
 159                 }
 160             }
 161             this.updateL3ForNeutronPort(neutronPort, isDelete);
 162         }
 163     }
 164
 165     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 166         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 167         if (!this.enabled)
 168             return;
 169     }
 170
 171     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 172                                                   final NeutronRouter_Interface neutronRouterInterface,
 173                                                   Action action) {
 174         logger.debug("Router interface {} got event {}. Subnet {}",
 175                      neutronRouterInterface.getPortUUID(),
 176                      action,
 177                      neutronRouterInterface.getSubnetUUID());
 178         if (!this.enabled)
 179             return;
 180
 181         final boolean isDelete = action == Action.DELETE;
 182
 183         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 184
 185         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 186         // see if they are affected by l3
 187         //
 188         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 189             boolean currPortIsInSameSubnet = false;
 190             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 191                 if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 192                     currPortIsInSameSubnet = true;
 193                     break;
 194                 }
 195             }
 196             if (currPortIsInSameSubnet == true) {
 197                 this.updateL3ForNeutronPort(neutronPort, isDelete);
 198             }
 199         }
 200     }
 201
 202     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 203                                              Action action) {
 204         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 205                      neutronFloatingIP.getFixedIPAddress(),
 206                      neutronFloatingIP.getFloatingIPAddress(),
 207                      neutronFloatingIP.getFloatingNetworkUUID());
 208         if (!this.enabled)
 209             return;
 210
 211         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 212     }
 213
 214     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 215         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 216         if (!this.enabled)
 217             return;
 218     }
 219
 220     //
 221     // Callbacks from OVSDB's southbound handler
 222     //
 223     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 224                                      Action action) {
 225         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 226                      action, node, intf.getName(), neutronNetwork);
 227         if (!this.enabled)
 228             return;
 229
 230         // See if there is an external uuid, so we can find the respective neutronPort
 231         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 232         if (externalIds == null) {
 233             return;
 234         }
 235         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 236         if (neutronPortId == null) {
 237             return;
 238         }
 239         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 240         if (neutronPort == null) {
 241             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 242                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 243             return;
 244         }
 245         this.handleNeutronPortEvent(neutronPort, action);
 246     }
 247
 248     //
 249     // Internal helpers
 250     //
 251     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 252
 253         final String networkUUID = neutronPort.getNetworkUUID();
 254         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 255
 256         // If there is no router interface handling the networkUUID, we are done
 257         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 258             return;
 259         }
 260
 261         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 262         // router interface are handled via handleNeutronRouterInterfaceEvent.
 263         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 264             return;
 265         }
 266
 267         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 268         final String providerSegmentationId = neutronNetwork != null ?
 269                                               neutronNetwork.getProviderSegmentationID() : null;
 270         final String tenantMac = neutronPort.getMacAddress();
 271
 272         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 273             tenantMac == null || tenantMac.isEmpty()) {
 274             return;  // done: go no further w/out all the info needed...
 275         }
 276
 277         final Action action = isDelete ? Action.DELETE : Action.ADD;
 278         List<Node> nodes = connectionService.getNodes();
 279         if (nodes.isEmpty()) {
 280             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 281         }
 282         for (Node node : nodes) {
 283             final Long dpid = getDpid(node);
 284             final Action actionForNode =
 285                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 286                     action : Action.DELETE;
 287             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 288                 final String tenantIpStr = neutronIP.getIpAddress();
 289                 if (tenantIpStr.isEmpty()) {
 290                     continue;
 291                 }
 292
 293                 // Configure L3 fwd
 294                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 295
 296                 // Configure distributed ARP responder
 297                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 298             }
 299         }
 300     }
 301
 302     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 303                                            String macAddress, String ipStr,
 304                                            Action actionForNode) {
 305         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 306         // will look at desired action for node.
 307         //
 308         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 309         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 310
 311         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 312             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 313                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 314             return;
 315         }
 316         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 317             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 318                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 319             return;
 320         }
 321
 322         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 323                                                        macAddress, ipStr, actionForNode);
 324         if (status.isSuccess()) {
 325             // Update cache
 326             if (actionForNode == Action.ADD) {
 327                 l3ForwardingCache.add(cacheKey);
 328             } else {
 329                 l3ForwardingCache.remove(cacheKey);
 330             }
 331         }
 332     }
 333
 334     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 335                                              String macAddress,
 336                                              String address,
 337                                              Action actionForNode) {
 338         Status status;
 339         try {
 340             InetAddress inetAddress = InetAddress.getByName(address);
 341             status = l3ForwardingProvider == null ?
 342                      new Status(StatusCode.SUCCESS) :
 343                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 344                                                                       inetAddress, macAddress, actionForNode);
 345         } catch (UnknownHostException e) {
 346             status = new Status(StatusCode.BADREQUEST);
 347         }
 348
 349         if (status.isSuccess()) {
 350             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 351                          l3ForwardingProvider == null ? "skipped" : "programmed",
 352                          macAddress, address, node, actionForNode);
 353         } else {
 354             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 355                          macAddress, address, node, actionForNode, status);
 356         }
 357         return status;
 358     }
 359
 360     // --
 361
 362     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 363                                                        Boolean isDelete) {
 364         Preconditions.checkNotNull(destNeutronRouterInterface);
 365
 366         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 367         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 368         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 369         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 370         final NeutronNetwork neutronNetwork = subnet != null ?
 371                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 372         final String destinationSegmentationId = neutronNetwork != null ?
 373                                                  neutronNetwork.getProviderSegmentationID() : null;
 374         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 375         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 376         final String cidr = subnet != null ? subnet.getCidr() : null;
 377         final int mask = getMaskLenFromCidr(cidr);
 378
 379         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 380                      destNeutronRouterInterface, isDelete);
 381
 382         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 383             cidr == null || cidr.isEmpty() ||
 384             macAddress == null || macAddress.isEmpty() ||
 385             ipList == null || ipList.isEmpty()) {
 386             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 387                          destinationSegmentationId, cidr, macAddress, ipList);
 388             return;  // done: go no further w/out all the info needed...
 389         }
 390
 391         final Action action = isDelete ? Action.DELETE : Action.ADD;
 392
 393         // Keep cache for finding router's mac from network uuid -- add
 394         //
 395         if (! isDelete) {
 396             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 397             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 398         }
 399
 400         List<Node> nodes = connectionService.getNodes();
 401         if (nodes.isEmpty()) {
 402             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 403         }
 404         for (Node node : nodes) {
 405             final Long dpid = getDpid(node);
 406             final Action actionForNode =
 407                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 408                     action : Action.DELETE;
 409
 410             for (Neutron_IPs neutronIP : ipList) {
 411                 final String ipStr = neutronIP.getIpAddress();
 412                 if (ipStr.isEmpty()) {
 413                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 414                                  node.getID(), ipStr);
 415                     continue;
 416                 }
 417
 418                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 419                 //
 420                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 421                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 422                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 423                                                               neutronNetwork, destinationSegmentationId,
 424                                                               macAddress, ipStr, mask, actionForNode,
 425                                                               true /*isReflexsive*/);
 426                 }
 427
 428                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 429             }
 430
 431             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 432             // for the external neutron networks.
 433             //
 434             {
 435                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 436                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 437                                                 cidr, actionForRewriteExclusion);
 438                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 439                                                 cidr, actionForRewriteExclusion);
 440             }
 441
 442             // Default route. For non-external subnet, make sure that there is none configured.
 443             //
 444             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 445                 final Action actionForNodeDefaultRoute =
 446                         isExternal ? actionForNode : Action.DELETE;
 447                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 448                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 449                                           actionForNodeDefaultRoute);
 450             }
 451         }
 452
 453         // Keep cache for finding router's mac from network uuid -- remove
 454         //
 455         if (isDelete) {
 456             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 457             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 458         }
 459     }
 460
 461     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 462                                                            final Long dpid,
 463                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 464                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 465                                                            final NeutronNetwork dstNeutronNetwork,
 466                                                            final String destinationSegmentationId,
 467                                                            final String dstMacAddress,
 468                                                            final String destIpStr,
 469                                                            final int destMask,
 470                                                            final Action actionForNode,
 471                                                            Boolean isReflexsive) {
 472         Preconditions.checkNotNull(srcNeutronRouterInterface);
 473         Preconditions.checkNotNull(dstNeutronRouterInterface);
 474
 475         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 476         if (sourceSubnetId == null) {
 477             logger.error("Could not get provider Subnet ID from router interface {}",
 478                          srcNeutronRouterInterface.getID());
 479             return;
 480         }
 481
 482         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 483         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 484         if (sourceNetworkId == null) {
 485             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 486             return;
 487         }
 488
 489         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 490         if (sourceNetwork == null) {
 491             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 492             return;
 493         }
 494
 495         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 496             // Isolate subnets from different tenants within the same router
 497             return;
 498         }
 499         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 500         if (sourceSegmentationId == null) {
 501             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 502             return;
 503         }
 504         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 505             // Skip 'self'
 506             return;
 507         }
 508
 509         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 510                                      dstMacAddress, destIpStr, destMask, actionForNode);
 511
 512         // Flip roles src->dst; dst->src
 513         if (isReflexsive) {
 514             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 515             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 516             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 517             final String cidr2 = sourceSubnet.getCidr();
 518             final int mask2 = getMaskLenFromCidr(cidr2);
 519
 520             if (cidr2 == null || cidr2.isEmpty() ||
 521                 macAddress2 == null || macAddress2.isEmpty() ||
 522                 ipList2 == null || ipList2.isEmpty()) {
 523                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 524                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 525                 return;  // done: go no further w/out all the info needed...
 526             }
 527
 528             for (Neutron_IPs neutronIP2 : ipList2) {
 529                 final String ipStr2 = neutronIP2.getIpAddress();
 530                 if (ipStr2.isEmpty()) {
 531                     continue;
 532                 }
 533                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 534                                                           srcNeutronRouterInterface,
 535                                                           sourceNetwork, sourceSegmentationId,
 536                                                           macAddress2, ipStr2, mask2, actionForNode,
 537                                                           false /*isReflexsive*/);
 538             }
 539         }
 540     }
 541
 542     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 543                                               String destinationSegmentationId,
 544                                               String macAddress, String ipStr, int mask,
 545                                               Action actionForNode) {
 546         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 547         // will look at desired action for node.
 548         //
 549         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 550                                 ipStr + "/" + Integer.toString(mask);
 551         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 552
 553         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 554             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 555                          "action {} is already done",
 556                          node.getNodeIDString(), sourceSegmentationId, destinationSegmentationId,
 557                          ipStr, mask, actionForNode);
 558             return;
 559         }
 560         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 561             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 562                          "action {} is already done",
 563                          node.getNodeIDString(), sourceSegmentationId, destinationSegmentationId,
 564                          ipStr, mask, actionForNode);
 565             return;
 566         }
 567
 568         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 569                                                           macAddress, ipStr, mask, actionForNode);
 570         if (status.isSuccess()) {
 571             // Update cache
 572             if (actionForNode == Action.ADD) {
 573                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 574                 routerInterfacesCache.add(cacheKey);
 575             } else {
 576                 // TODO: multiTenantAwareRouter.removeInterface(...);
 577                 routerInterfacesCache.remove(cacheKey);
 578             }
 579         }
 580     }
 581
 582     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 583                                                 String destinationSegmentationId,
 584                                                 String macAddress,
 585                                                 String address, int mask,
 586                                                 Action actionForNode) {
 587         Status status;
 588         try {
 589             InetAddress inetAddress = InetAddress.getByName(address);
 590             status = routingProvider == null ?
 591                      new Status(StatusCode.SUCCESS) :
 592                      routingProvider.programRouterInterface(node, dpid, sourceSegmentationId, destinationSegmentationId,
 593                                                             macAddress, inetAddress, mask, actionForNode);
 594         } catch (UnknownHostException e) {
 595             status = new Status(StatusCode.BADREQUEST);
 596         }
 597
 598         if (status.isSuccess()) {
 599             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 600                          routingProvider == null ? "skipped" : "programmed",
 601                          macAddress, address, mask, node, actionForNode);
 602         } else {
 603             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 604                          macAddress, address, mask, node, actionForNode, status);
 605         }
 606         return status;
 607     }
 608
 609     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 610                                         String macAddress, String ipStr,
 611                                         Action actionForNode) {
 612         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 613         // will look at desired action for node.
 614         //
 615         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 616         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 617
 618         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 619             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 620                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 621             return;
 622         }
 623         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 624             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 625                          node.getNodeIDString(), providerSegmentationId, macAddress, ipStr, actionForNode);
 626             return;
 627         }
 628
 629         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 630                                                     macAddress, ipStr, actionForNode);
 631         if (status.isSuccess()) {
 632             // Update cache
 633             if (actionForNode == Action.ADD) {
 634                 staticArpEntryCache.add(cacheKey);
 635             } else {
 636                 staticArpEntryCache.remove(cacheKey);
 637             }
 638         }
 639     }
 640
 641     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 642                                                 String macAddress,
 643                                                 String address,
 644                                                 Action actionForNode) {
 645         Status status;
 646         try {
 647             InetAddress inetAddress = InetAddress.getByName(address);
 648             status = arpProvider == null ?
 649                      new Status(StatusCode.SUCCESS) :
 650                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 651                                                        macAddress, inetAddress, actionForNode);
 652         } catch (UnknownHostException e) {
 653             status = new Status(StatusCode.BADREQUEST);
 654         }
 655
 656         if (status.isSuccess()) {
 657             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 658                          arpProvider == null ? "skipped" : "programmed",
 659                          macAddress, address, node, actionForNode);
 660         } else {
 661             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 662                          macAddress, address, node, actionForNode, status);
 663         }
 664         return status;
 665     }
 666
 667     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 668                                                  final boolean isInbound, String cidr,
 669                                                  Action actionForRewriteExclusion) {
 670         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 671         // will look at desired action for node.
 672         //
 673         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 674         final Boolean isProgrammed = isInbound ?
 675                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 676                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 677
 678         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 679             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 680                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 681                          actionForRewriteExclusion);
 682             return;
 683         }
 684         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 685             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 686                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 687                          actionForRewriteExclusion);
 688             return;
 689         }
 690
 691         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 692                                                              isInbound, actionForRewriteExclusion);
 693         if (status.isSuccess()) {
 694             // Update cache
 695             if (actionForRewriteExclusion == Action.ADD) {
 696                 if (isInbound) {
 697                     inboundIpRewriteExclusionCache.add(cacheKey);
 698                 } else {
 699                     outboundIpRewriteExclusionCache.add(cacheKey);
 700                 }
 701             } else {
 702                 if (isInbound) {
 703                     inboundIpRewriteExclusionCache.remove(cacheKey);
 704                 } else {
 705                     outboundIpRewriteExclusionCache.remove(cacheKey);
 706                 }
 707             }
 708         }
 709     }
 710
 711     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 712                                                    final boolean isInbound, Action actionForNode) {
 713         Status status;
 714         if (isInbound) {
 715             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 716                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 717                                                                   actionForNode);
 718         } else {
 719             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 720                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 721                                                                    actionForNode);
 722         }
 723
 724         if (status.isSuccess()) {
 725             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 726             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 727                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 728                          cidr, node, actionForNode);
 729         } else {
 730             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 731                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 732         }
 733         return status;
 734     }
 735
 736     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 737                                            String defaultGatewayMacAddress, String gatewayIp,
 738                                            Action actionForNodeDefaultRoute) {
 739         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 740         // will look at desired action for node.
 741         //
 742         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 743         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 744
 745         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 746             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 747                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 748                          actionForNodeDefaultRoute);
 749             return;
 750         }
 751         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 752             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 753                          node.getNodeIDString(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 754                          actionForNodeDefaultRoute);
 755             return;
 756         }
 757
 758         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 759                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 760         if (status.isSuccess()) {
 761             // Update cache
 762             if (actionForNodeDefaultRoute == Action.ADD) {
 763                 defaultRouteCache.add(cacheKey);
 764             } else {
 765                 defaultRouteCache.remove(cacheKey);
 766             }
 767         }
 768     }
 769
 770     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 771                                           String defaultGatewayMacAddress,
 772                                           String gatewayIp,
 773                                           Action actionForNodeDefaultRoute) {
 774         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 775         if (defaultGatewayMacAddress == null) {
 776             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 777                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 778             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 779         }
 780
 781         Status status;
 782         try {
 783             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 784             status = routingProvider == null ?
 785                      new Status(StatusCode.SUCCESS) :
 786                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 787                                                               defaultGatewayMacAddress, inetAddress,
 788                                                               actionForNodeDefaultRoute);
 789         } catch (UnknownHostException e) {
 790             status = new Status(StatusCode.BADREQUEST);
 791         }
 792
 793         if (status.isSuccess()) {
 794             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 795                          routingProvider == null ? "skipped" : "programmed",
 796                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 797         } else {
 798             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 799                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 800         }
 801         return status;
 802     }
 803
 804     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 805         Preconditions.checkNotNull(neutronFloatingIP);
 806
 807         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 808         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 809
 810         // If there is no router interface handling the networkUUID, we are done
 811         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 812             return;
 813         }
 814
 815         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 816         final String providerSegmentationId = neutronNetwork != null ?
 817                                               neutronNetwork.getProviderSegmentationID() : null;
 818         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 819         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 820
 821         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 822             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 823             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 824             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 825             return;  // done: go no further w/out all the info needed...
 826         }
 827
 828         final Action action = isDelete ? Action.DELETE : Action.ADD;
 829         List<Node> nodes = connectionService.getNodes();
 830         if (nodes.isEmpty()) {
 831             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 832         }
 833         for (Node node : nodes) {
 834             final Long dpid = getDpid(node);
 835             final Action actionForNode =
 836                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 837                     action : Action.DELETE;
 838
 839             // Rewrite from float to fixed and vice-versa
 840             //
 841             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 842                                    floatingIpAddress, fixedIPAddress, actionForNode);
 843             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 844                                    fixedIPAddress, floatingIpAddress, actionForNode);
 845
 846             // Respond to arps for the floating ip address
 847             //
 848             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 849                                    actionForNode);
 850         }
 851     }
 852
 853     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 854                                         final boolean isInbound,
 855                                         String matchAddress, String rewriteAddress,
 856                                         Action actionForNode) {
 857         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 858         // will look at desired action for node.
 859         //
 860         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 861                                 matchAddress + ":" + rewriteAddress;
 862         final Boolean isProgrammed = isInbound ?
 863                                      inboundIpRewriteCache.contains(cacheKey) :
 864                                      outboundIpRewriteCache.contains(cacheKey);
 865
 866         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 867             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 868                          " is already done",
 869                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 870                          matchAddress, rewriteAddress, actionForNode);
 871             return;
 872         }
 873         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 874             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 875                          " is already done",
 876                          node.getNodeIDString(), providerSegmentationId, isInbound ? "inbound": "outbound",
 877                          matchAddress, rewriteAddress, actionForNode);
 878             return;
 879         }
 880
 881         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 882                                                     matchAddress, rewriteAddress, actionForNode);
 883         if (status.isSuccess()) {
 884             // Update cache
 885             if (actionForNode == Action.ADD) {
 886                 if (isInbound) {
 887                     inboundIpRewriteCache.add(cacheKey);
 888                 } else {
 889                     outboundIpRewriteCache.add(cacheKey);
 890                 }
 891             } else {
 892                 if (isInbound) {
 893                     inboundIpRewriteCache.remove(cacheKey);
 894                 } else {
 895                     outboundIpRewriteCache.remove(cacheKey);
 896                 }
 897             }
 898         }
 899     }
 900
 901     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 902                                           final boolean isInbound,
 903                                           String matchAddress, String rewriteAddress,
 904                                           Action actionForNode) {
 905         Status status;
 906         try {
 907             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 908             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 909             if (isInbound) {
 910                 status = inboundNatProvider == null ?
 911                          new Status(StatusCode.SUCCESS) :
 912                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 913                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 914             } else {
 915                 status = outboundNatProvider == null ?
 916                          new Status(StatusCode.SUCCESS) :
 917                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 918                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 919             }
 920         } catch (UnknownHostException e) {
 921             status = new Status(StatusCode.BADREQUEST);
 922         }
 923
 924         if (status.isSuccess()) {
 925             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 926             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 927                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 928                          matchAddress, rewriteAddress, node, actionForNode);
 929         } else {
 930             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 931                          (isInbound ? "inbound" : "outbound"),
 932                          matchAddress, rewriteAddress, node, actionForNode, status);
 933         }
 934         return status;
 935     }
 936
 937     //
 938     // More Internals
 939     //
 940
 941     private int getMaskLenFromCidr(String cidr) {
 942         if (cidr == null) return 0;
 943         String[] splits = cidr.split("/");
 944         if (splits.length != 2) return 0;
 945
 946         int result;
 947         try {
 948             result = Integer.parseInt(splits[1].trim());
 949         }
 950         catch (NumberFormatException nfe)
 951         {
 952             result = 0;
 953         }
 954         return result;
 955     }
 956
 957     private Long getDpid (Node node) {
 958         Preconditions.checkNotNull(ovsdbConfigurationService);
 959
 960         String bridgeName = configurationService.getIntegrationBridgeName();
 961         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 962         if (bridgeUuid == null) {
 963             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 964             return 0L;
 965         }
 966
 967         try {
 968             Row bridgeRow =  ovsdbConfigurationService
 969                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 970             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 971             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 972             if (dpids == null || dpids.size() == 0) return 0L;
 973             return HexEncode.stringToLong((String) dpids.toArray()[0]);
 974         } catch (Exception e) {
 975             logger.error("Error finding Bridge's OF DPID", e);
 976             return 0L;
 977         }
 978     }
 979
 980     private String getInternalBridgeUUID (Node node, String bridgeName) {
 981         Preconditions.checkNotNull(ovsdbConfigurationService);
 982         try {
 983             Map<String, Row> bridgeTable =
 984                     ovsdbConfigurationService.getRows(node,
 985                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 986             if (bridgeTable == null) return null;
 987             for (String key : bridgeTable.keySet()) {
 988                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 989                 if (bridge.getName().equals(bridgeName)) return key;
 990             }
 991         } catch (Exception e) {
 992             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 993         }
 994         return null;
 995     }
 996 }
 997
 998