openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import java.util.ArrayList;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.lib.notation.Row;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  27 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  28 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  29 import org.opendaylight.ovsdb.utils.mdsal.node.StringConvertor;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  31 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  32
  33 import com.google.common.base.Preconditions;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.net.InetAddress;
  38 import java.net.UnknownHostException;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.List;
  42 import java.util.Map;
  43 import java.util.Set;
  44
  45 /**
  46  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  47  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  48  * as well as the multi-tenant router forwarding provider.
  49  */
  50 public class NeutronL3Adapter {
  51
  52     /**
  53      * Logger instance.
  54      */
  55     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  56
  57     // The implementation for each of these services is resolved by the OSGi Service Manager
  58     private volatile ConfigurationService configurationService;
  59     private volatile TenantNetworkManager tenantNetworkManager;
  60     /* TODO SB_MIGRATION */
  61     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  62     private volatile OvsdbConnectionService connectionService;
  63     private volatile MdsalConsumer mdsalConsumer;
  64     private volatile INeutronNetworkCRUD neutronNetworkCache;
  65     private volatile INeutronSubnetCRUD neutronSubnetCache;
  66     private volatile INeutronPortCRUD neutronPortCache;
  67     private volatile L3ForwardingProvider l3ForwardingProvider;
  68     private volatile InboundNatProvider inboundNatProvider;
  69     private volatile OutboundNatProvider outboundNatProvider;
  70     private volatile ArpProvider arpProvider;
  71     private volatile RoutingProvider routingProvider;
  72
  73     private Set<String> inboundIpRewriteCache;
  74     private Set<String> outboundIpRewriteCache;
  75     private Set<String> inboundIpRewriteExclusionCache;
  76     private Set<String> outboundIpRewriteExclusionCache;
  77     private Set<String> routerInterfacesCache;
  78     private Set<String> staticArpEntryCache;
  79     private Set<String> l3ForwardingCache;
  80     private Set<String> defaultRouteCache;
  81     private Map<String, String> networkIdToRouterMacCache;
  82     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  83     private Boolean enabled = false;
  84
  85     void init() {
  86         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  87         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  88             this.inboundIpRewriteCache = new HashSet<>();
  89             this.outboundIpRewriteCache = new HashSet<>();
  90             this.inboundIpRewriteExclusionCache = new HashSet<>();
  91             this.outboundIpRewriteExclusionCache = new HashSet<>();
  92             this.routerInterfacesCache = new HashSet<>();
  93             this.staticArpEntryCache = new HashSet<>();
  94             this.l3ForwardingCache = new HashSet<>();
  95             this.defaultRouteCache = new HashSet<>();
  96             this.networkIdToRouterMacCache = new HashMap<>();
  97             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  98
  99             this.enabled = true;
 100             logger.info("OVSDB L3 forwarding is enabled");
 101         } else {
 102             logger.debug("OVSDB L3 forwarding is disabled");
 103         }
 104     }
 105
 106     //
 107     // Callbacks from OVSDB's northbound handlers
 108     //
 109
 110     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 111         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 112         if (!this.enabled)
 113             return;
 114     }
 115
 116     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 117         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 118         if (!this.enabled)
 119             return;
 120
 121         final boolean isDelete = action == Action.DELETE;
 122
 123         // Treat the port event as a router interface event if the port belongs to router. This is a
 124         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 125         //
 126         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 127             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 128                 NeutronRouter_Interface neutronRouterInterface =
 129                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 130                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 131                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 132
 133                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 134             }
 135         } else {
 136             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 137             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 138             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 139             // there.
 140             //
 141             if (!isDelete) {
 142                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 143                     NeutronRouter_Interface neutronRouterInterface =
 144                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 145                     if (neutronRouterInterface != null) {
 146                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 147                     }
 148                 }
 149             }
 150             this.updateL3ForNeutronPort(neutronPort, isDelete);
 151         }
 152     }
 153
 154     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 155         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 156         if (!this.enabled)
 157             return;
 158     }
 159
 160     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 161                                                   final NeutronRouter_Interface neutronRouterInterface,
 162                                                   Action action) {
 163         logger.debug("Router interface {} got event {}. Subnet {}",
 164                      neutronRouterInterface.getPortUUID(),
 165                      action,
 166                      neutronRouterInterface.getSubnetUUID());
 167         if (!this.enabled)
 168             return;
 169
 170         final boolean isDelete = action == Action.DELETE;
 171
 172         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 173
 174         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 175         // see if they are affected by l3
 176         //
 177         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 178             boolean currPortShouldBeDeleted = false;
 179             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 180             if (isDelete) {
 181                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 182                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 183                         currPortShouldBeDeleted = true;
 184                         break;
 185                     }
 186                 }
 187             }
 188             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 189         }
 190     }
 191
 192     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 193                                              Action action) {
 194         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 195                      neutronFloatingIP.getFixedIPAddress(),
 196                      neutronFloatingIP.getFloatingIPAddress(),
 197                      neutronFloatingIP.getFloatingNetworkUUID());
 198         if (!this.enabled)
 199             return;
 200
 201         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 202     }
 203
 204     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 205         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 206         if (!this.enabled)
 207             return;
 208     }
 209
 210     //
 211     // Callbacks from OVSDB's southbound handler
 212     //
 213     public void handleInterfaceEvent(final Node node, final OvsdbTerminationPointAugmentation intf,
 214                                      final NeutronNetwork neutronNetwork, Action action) {
 215         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 216                      action, node, intf.getName(), neutronNetwork);
 217         if (!this.enabled)
 218             return;
 219
 220         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 221         if (neutronPort != null) {
 222             this.handleNeutronPortEvent(neutronPort, action);
 223         }
 224     }
 225
 226     //
 227     // Internal helpers
 228     //
 229     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 230
 231         final String networkUUID = neutronPort.getNetworkUUID();
 232         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 233
 234         // If there is no router interface handling the networkUUID, we are done
 235         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 236             return;
 237         }
 238
 239         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 240         // router interface are handled via handleNeutronRouterInterfaceEvent.
 241         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 242             return;
 243         }
 244
 245         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 246         final String providerSegmentationId = neutronNetwork != null ?
 247                                               neutronNetwork.getProviderSegmentationID() : null;
 248         final String tenantMac = neutronPort.getMacAddress();
 249
 250         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 251             tenantMac == null || tenantMac.isEmpty()) {
 252             return;  // done: go no further w/out all the info needed...
 253         }
 254
 255         final Action action = isDelete ? Action.DELETE : Action.ADD;
 256         /* TODO SB_MIGRATION
 257         List<Node> nodes = connectionService.getNodes();
 258         */
 259         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 260         if (nodes.isEmpty()) {
 261             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 262         }
 263         for (Node node : nodes) {
 264             final Long dpid = getDpid(node);
 265             final boolean tenantNetworkPresentInNode =
 266                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 267             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 268                 final String tenantIpStr = neutronIP.getIpAddress();
 269                 if (tenantIpStr.isEmpty()) {
 270                     continue;
 271                 }
 272
 273                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 274                 // still needed when routing to subnets non-local to node (bug 2076).
 275                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 276
 277                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 278                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 279                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 280             }
 281         }
 282     }
 283
 284     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 285                                            String macAddress, String ipStr,
 286                                            Action actionForNode) {
 287         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 288         // will look at desired action for node.
 289         //
 290         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 291         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 292
 293         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 294             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 295                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 296             return;
 297         }
 298         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 299             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 300                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 301             return;
 302         }
 303
 304         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 305                                                        macAddress, ipStr, actionForNode);
 306         if (status.isSuccess()) {
 307             // Update cache
 308             if (actionForNode == Action.ADD) {
 309                 l3ForwardingCache.add(cacheKey);
 310             } else {
 311                 l3ForwardingCache.remove(cacheKey);
 312             }
 313         }
 314     }
 315
 316     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 317                                              String macAddress,
 318                                              String address,
 319                                              Action actionForNode) {
 320         Status status;
 321         try {
 322             InetAddress inetAddress = InetAddress.getByName(address);
 323             status = l3ForwardingProvider == null ?
 324                      new Status(StatusCode.SUCCESS) :
 325                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 326                                                                       inetAddress, macAddress, actionForNode);
 327         } catch (UnknownHostException e) {
 328             status = new Status(StatusCode.BADREQUEST);
 329         }
 330
 331         if (status.isSuccess()) {
 332             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 333                          l3ForwardingProvider == null ? "skipped" : "programmed",
 334                          macAddress, address, node, actionForNode);
 335         } else {
 336             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 337                          macAddress, address, node, actionForNode, status);
 338         }
 339         return status;
 340     }
 341
 342     // --
 343
 344     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 345                                                        Boolean isDelete) {
 346         Preconditions.checkNotNull(destNeutronRouterInterface);
 347
 348         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 349         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 350         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 351         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 352         final NeutronNetwork neutronNetwork = subnet != null ?
 353                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 354         final String destinationSegmentationId = neutronNetwork != null ?
 355                                                  neutronNetwork.getProviderSegmentationID() : null;
 356         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 357         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 358         final String cidr = subnet != null ? subnet.getCidr() : null;
 359         final int mask = getMaskLenFromCidr(cidr);
 360
 361         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 362                      destNeutronRouterInterface, isDelete);
 363
 364         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 365             cidr == null || cidr.isEmpty() ||
 366             macAddress == null || macAddress.isEmpty() ||
 367             ipList == null || ipList.isEmpty()) {
 368             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 369                          destinationSegmentationId, cidr, macAddress, ipList);
 370             return;  // done: go no further w/out all the info needed...
 371         }
 372
 373         final Action action = isDelete ? Action.DELETE : Action.ADD;
 374
 375         // Keep cache for finding router's mac from network uuid -- add
 376         //
 377         if (! isDelete) {
 378             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 379             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 380         }
 381
 382         /* TODO SB_MIGRATION
 383         List<Node> nodes = connectionService.getNodes();*/
 384         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 385         if (nodes.isEmpty()) {
 386             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 387         }
 388         for (Node node : nodes) {
 389             final Long dpid = getDpid(node);
 390             final Action actionForNode =
 391                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 392                     action : Action.DELETE;
 393
 394             for (Neutron_IPs neutronIP : ipList) {
 395                 final String ipStr = neutronIP.getIpAddress();
 396                 if (ipStr.isEmpty()) {
 397                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 398                             node.getNodeId().getValue(), ipStr);
 399                     continue;
 400                 }
 401
 402                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 403                 //
 404                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 405                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 406                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 407                                                               neutronNetwork, destinationSegmentationId,
 408                                                               macAddress, ipStr, mask, actionForNode,
 409                                                               true /*isReflexsive*/);
 410                 }
 411
 412                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 413             }
 414
 415             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 416             // for the external neutron networks.
 417             //
 418             {
 419                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 420                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 421                                                 cidr, actionForRewriteExclusion);
 422                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 423                                                 cidr, actionForRewriteExclusion);
 424             }
 425
 426             // Default route. For non-external subnet, make sure that there is none configured.
 427             //
 428             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 429                 final Action actionForNodeDefaultRoute =
 430                         isExternal ? actionForNode : Action.DELETE;
 431                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 432                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 433                                           actionForNodeDefaultRoute);
 434             }
 435         }
 436
 437         // Keep cache for finding router's mac from network uuid -- remove
 438         //
 439         if (isDelete) {
 440             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 441             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 442         }
 443     }
 444
 445     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 446                                                            final Long dpid,
 447                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 448                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 449                                                            final NeutronNetwork dstNeutronNetwork,
 450                                                            final String destinationSegmentationId,
 451                                                            final String dstMacAddress,
 452                                                            final String destIpStr,
 453                                                            final int destMask,
 454                                                            final Action actionForNode,
 455                                                            Boolean isReflexsive) {
 456         Preconditions.checkNotNull(srcNeutronRouterInterface);
 457         Preconditions.checkNotNull(dstNeutronRouterInterface);
 458
 459         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 460         if (sourceSubnetId == null) {
 461             logger.error("Could not get provider Subnet ID from router interface {}",
 462                          srcNeutronRouterInterface.getID());
 463             return;
 464         }
 465
 466         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 467         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 468         if (sourceNetworkId == null) {
 469             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 470             return;
 471         }
 472
 473         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 474         if (sourceNetwork == null) {
 475             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 476             return;
 477         }
 478
 479         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 480             // Isolate subnets from different tenants within the same router
 481             return;
 482         }
 483         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 484         if (sourceSegmentationId == null) {
 485             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 486             return;
 487         }
 488         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 489             // Skip 'self'
 490             return;
 491         }
 492
 493         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 494                                      dstMacAddress, destIpStr, destMask, actionForNode);
 495
 496         // Flip roles src->dst; dst->src
 497         if (isReflexsive) {
 498             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 499             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 500             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 501             final String cidr2 = sourceSubnet.getCidr();
 502             final int mask2 = getMaskLenFromCidr(cidr2);
 503
 504             if (cidr2 == null || cidr2.isEmpty() ||
 505                 macAddress2 == null || macAddress2.isEmpty() ||
 506                 ipList2 == null || ipList2.isEmpty()) {
 507                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 508                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 509                 return;  // done: go no further w/out all the info needed...
 510             }
 511
 512             for (Neutron_IPs neutronIP2 : ipList2) {
 513                 final String ipStr2 = neutronIP2.getIpAddress();
 514                 if (ipStr2.isEmpty()) {
 515                     continue;
 516                 }
 517                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 518                                                           srcNeutronRouterInterface,
 519                                                           sourceNetwork, sourceSegmentationId,
 520                                                           macAddress2, ipStr2, mask2, actionForNode,
 521                                                           false /*isReflexsive*/);
 522             }
 523         }
 524     }
 525
 526     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 527                                               String destinationSegmentationId,
 528                                               String macAddress, String ipStr, int mask,
 529                                               Action actionForNode) {
 530         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 531         // will look at desired action for node.
 532         //
 533         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 534                                 ipStr + "/" + Integer.toString(mask);
 535         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 536
 537         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 538             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 539                          "action {} is already done",
 540                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 541                          ipStr, mask, actionForNode);
 542             return;
 543         }
 544         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 545             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 546                          "action {} is already done",
 547                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 548                          ipStr, mask, actionForNode);
 549             return;
 550         }
 551
 552         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 553                                                           macAddress, ipStr, mask, actionForNode);
 554         if (status.isSuccess()) {
 555             // Update cache
 556             if (actionForNode == Action.ADD) {
 557                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 558                 routerInterfacesCache.add(cacheKey);
 559             } else {
 560                 // TODO: multiTenantAwareRouter.removeInterface(...);
 561                 routerInterfacesCache.remove(cacheKey);
 562             }
 563         }
 564     }
 565
 566     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 567                                                 String destinationSegmentationId,
 568                                                 String macAddress,
 569                                                 String address, int mask,
 570                                                 Action actionForNode) {
 571         Status status;
 572         try {
 573             InetAddress inetAddress = InetAddress.getByName(address);
 574             status = routingProvider == null ?
 575                      new Status(StatusCode.SUCCESS) :
 576                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 577                                                             macAddress, inetAddress, mask, actionForNode);
 578         } catch (UnknownHostException e) {
 579             status = new Status(StatusCode.BADREQUEST);
 580         }
 581
 582         if (status.isSuccess()) {
 583             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 584                          routingProvider == null ? "skipped" : "programmed",
 585                          macAddress, address, mask, node, actionForNode);
 586         } else {
 587             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 588                          macAddress, address, mask, node, actionForNode, status);
 589         }
 590         return status;
 591     }
 592
 593     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 594                                         String macAddress, String ipStr,
 595                                         Action actionForNode) {
 596         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 597         // will look at desired action for node.
 598         //
 599         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 600         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 601
 602         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 603             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 604                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 605             return;
 606         }
 607         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 608             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 609                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 610             return;
 611         }
 612
 613         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 614                                                     macAddress, ipStr, actionForNode);
 615         if (status.isSuccess()) {
 616             // Update cache
 617             if (actionForNode == Action.ADD) {
 618                 staticArpEntryCache.add(cacheKey);
 619             } else {
 620                 staticArpEntryCache.remove(cacheKey);
 621             }
 622         }
 623     }
 624
 625     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 626                                                 String macAddress,
 627                                                 String address,
 628                                                 Action actionForNode) {
 629         Status status;
 630         try {
 631             InetAddress inetAddress = InetAddress.getByName(address);
 632             status = arpProvider == null ?
 633                      new Status(StatusCode.SUCCESS) :
 634                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 635                                                        macAddress, inetAddress, actionForNode);
 636         } catch (UnknownHostException e) {
 637             status = new Status(StatusCode.BADREQUEST);
 638         }
 639
 640         if (status.isSuccess()) {
 641             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 642                          arpProvider == null ? "skipped" : "programmed",
 643                          macAddress, address, node, actionForNode);
 644         } else {
 645             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 646                          macAddress, address, node, actionForNode, status);
 647         }
 648         return status;
 649     }
 650
 651     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 652                                                  final boolean isInbound, String cidr,
 653                                                  Action actionForRewriteExclusion) {
 654         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 655         // will look at desired action for node.
 656         //
 657         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 658         final Boolean isProgrammed = isInbound ?
 659                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 660                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 661
 662         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 663             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 664                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 665                          actionForRewriteExclusion);
 666             return;
 667         }
 668         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 669             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 670                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 671                          actionForRewriteExclusion);
 672             return;
 673         }
 674
 675         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 676                                                              isInbound, actionForRewriteExclusion);
 677         if (status.isSuccess()) {
 678             // Update cache
 679             if (actionForRewriteExclusion == Action.ADD) {
 680                 if (isInbound) {
 681                     inboundIpRewriteExclusionCache.add(cacheKey);
 682                 } else {
 683                     outboundIpRewriteExclusionCache.add(cacheKey);
 684                 }
 685             } else {
 686                 if (isInbound) {
 687                     inboundIpRewriteExclusionCache.remove(cacheKey);
 688                 } else {
 689                     outboundIpRewriteExclusionCache.remove(cacheKey);
 690                 }
 691             }
 692         }
 693     }
 694
 695     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 696                                                    final boolean isInbound, Action actionForNode) {
 697         Status status;
 698         if (isInbound) {
 699             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 700                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 701                                                                   actionForNode);
 702         } else {
 703             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 704                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 705                                                                    actionForNode);
 706         }
 707
 708         if (status.isSuccess()) {
 709             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 710             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 711                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 712                          cidr, node, actionForNode);
 713         } else {
 714             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 715                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 716         }
 717         return status;
 718     }
 719
 720     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 721                                            String defaultGatewayMacAddress, String gatewayIp,
 722                                            Action actionForNodeDefaultRoute) {
 723         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 724         // will look at desired action for node.
 725         //
 726         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 727         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 728
 729         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 730             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 731                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 732                          actionForNodeDefaultRoute);
 733             return;
 734         }
 735         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 736             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 737                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 738                          actionForNodeDefaultRoute);
 739             return;
 740         }
 741
 742         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 743                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 744         if (status.isSuccess()) {
 745             // Update cache
 746             if (actionForNodeDefaultRoute == Action.ADD) {
 747                 defaultRouteCache.add(cacheKey);
 748             } else {
 749                 defaultRouteCache.remove(cacheKey);
 750             }
 751         }
 752     }
 753
 754     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 755                                           String defaultGatewayMacAddress,
 756                                           String gatewayIp,
 757                                           Action actionForNodeDefaultRoute) {
 758         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 759         if (defaultGatewayMacAddress == null) {
 760             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 761                          gatewayIp, node, actionForNodeDefaultRoute);
 762             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 763         }
 764
 765         Status status;
 766         try {
 767             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 768             status = routingProvider == null ?
 769                      new Status(StatusCode.SUCCESS) :
 770                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 771                                                               defaultGatewayMacAddress, inetAddress,
 772                                                               actionForNodeDefaultRoute);
 773         } catch (UnknownHostException e) {
 774             status = new Status(StatusCode.BADREQUEST);
 775         }
 776
 777         if (status.isSuccess()) {
 778             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 779                          routingProvider == null ? "skipped" : "programmed",
 780                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 781         } else {
 782             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 783                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 784         }
 785         return status;
 786     }
 787
 788     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 789         Preconditions.checkNotNull(neutronFloatingIP);
 790
 791         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 792         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 793
 794         // If there is no router interface handling the networkUUID, we are done
 795         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 796             return;
 797         }
 798
 799         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 800         final String providerSegmentationId = neutronNetwork != null ?
 801                                               neutronNetwork.getProviderSegmentationID() : null;
 802         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 803         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 804
 805         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 806             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 807             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 808             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 809             return;  // done: go no further w/out all the info needed...
 810         }
 811
 812         final Action action = isDelete ? Action.DELETE : Action.ADD;
 813         /* TODO SB_MIGRATION
 814         List<Node> nodes = connectionService.getNodes();*/
 815         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 816         if (nodes.isEmpty()) {
 817             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 818         }
 819         for (Node node : nodes) {
 820             final Long dpid = getDpid(node);
 821             final Action actionForNode =
 822                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 823                     action : Action.DELETE;
 824
 825             // Rewrite from float to fixed and vice-versa
 826             //
 827             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 828                                    floatingIpAddress, fixedIPAddress, actionForNode);
 829             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 830                                    fixedIPAddress, floatingIpAddress, actionForNode);
 831
 832             // Respond to arps for the floating ip address
 833             //
 834             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 835                                    actionForNode);
 836         }
 837     }
 838
 839     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 840                                         final boolean isInbound,
 841                                         String matchAddress, String rewriteAddress,
 842                                         Action actionForNode) {
 843         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 844         // will look at desired action for node.
 845         //
 846         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 847                                 matchAddress + ":" + rewriteAddress;
 848         final Boolean isProgrammed = isInbound ?
 849                                      inboundIpRewriteCache.contains(cacheKey) :
 850                                      outboundIpRewriteCache.contains(cacheKey);
 851
 852         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 853             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 854                          " is already done",
 855                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 856                          matchAddress, rewriteAddress, actionForNode);
 857             return;
 858         }
 859         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 860             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 861                          " is already done",
 862                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 863                          matchAddress, rewriteAddress, actionForNode);
 864             return;
 865         }
 866
 867         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 868                                                     matchAddress, rewriteAddress, actionForNode);
 869         if (status.isSuccess()) {
 870             // Update cache
 871             if (actionForNode == Action.ADD) {
 872                 if (isInbound) {
 873                     inboundIpRewriteCache.add(cacheKey);
 874                 } else {
 875                     outboundIpRewriteCache.add(cacheKey);
 876                 }
 877             } else {
 878                 if (isInbound) {
 879                     inboundIpRewriteCache.remove(cacheKey);
 880                 } else {
 881                     outboundIpRewriteCache.remove(cacheKey);
 882                 }
 883             }
 884         }
 885     }
 886
 887     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 888                                           final boolean isInbound,
 889                                           String matchAddress, String rewriteAddress,
 890                                           Action actionForNode) {
 891         Status status;
 892         try {
 893             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 894             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 895             if (isInbound) {
 896                 status = inboundNatProvider == null ?
 897                          new Status(StatusCode.SUCCESS) :
 898                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 899                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 900             } else {
 901                 status = outboundNatProvider == null ?
 902                          new Status(StatusCode.SUCCESS) :
 903                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 904                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 905             }
 906         } catch (UnknownHostException e) {
 907             status = new Status(StatusCode.BADREQUEST);
 908         }
 909
 910         if (status.isSuccess()) {
 911             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 912             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 913                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 914                          matchAddress, rewriteAddress, node, actionForNode);
 915         } else {
 916             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 917                          (isInbound ? "inbound" : "outbound"),
 918                          matchAddress, rewriteAddress, node, actionForNode, status);
 919         }
 920         return status;
 921     }
 922
 923     //
 924     // More Internals
 925     //
 926
 927     private int getMaskLenFromCidr(String cidr) {
 928         if (cidr == null) return 0;
 929         String[] splits = cidr.split("/");
 930         if (splits.length != 2) return 0;
 931
 932         int result;
 933         try {
 934             result = Integer.parseInt(splits[1].trim());
 935         }
 936         catch (NumberFormatException nfe)
 937         {
 938             result = 0;
 939         }
 940         return result;
 941     }
 942
 943     private Long getDpid (Node node) {
 944         /* TODO SB_MIGRATION */
 945         Preconditions.checkNotNull(ovsdbConfigurationService);
 946
 947         String bridgeName = configurationService.getIntegrationBridgeName();
 948         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 949         if (bridgeUuid == null) {
 950             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 951             return 0L;
 952         }
 953
 954         try {
 955             Row bridgeRow =  ovsdbConfigurationService
 956                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 957             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 958             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 959             if (dpids == null || dpids.size() == 0) return 0L;
 960             return StringConvertor.dpidStringToLong((String) dpids.toArray()[0]);
 961         } catch (Exception e) {
 962             logger.error("Error finding Bridge's OF DPID", e);
 963             return 0L;
 964         }
 965     }
 966
 967     private String getInternalBridgeUUID (Node node, String bridgeName) {
 968         /* TODO SB_MIGRATION */
 969         Preconditions.checkNotNull(ovsdbConfigurationService);
 970         try {
 971             Map<String, Row> bridgeTable =
 972                     ovsdbConfigurationService.getRows(node,
 973                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 974             if (bridgeTable == null) return null;
 975             for (String key : bridgeTable.keySet()) {
 976                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 977                 if (bridge.getName().equals(bridgeName)) return key;
 978             }
 979         } catch (Exception e) {
 980             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 981         }
 982         return null;
 983     }
 984 }