openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import java.util.ArrayList;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.lib.notation.Row;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  27 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  28 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  29 import org.opendaylight.ovsdb.utils.mdsal.node.StringConvertor;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import org.slf4j.Logger;
  34 import org.slf4j.LoggerFactory;
  35
  36 import java.net.InetAddress;
  37 import java.net.UnknownHostException;
  38 import java.util.HashMap;
  39 import java.util.HashSet;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.Set;
  43
  44 /**
  45  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  46  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  47  * as well as the multi-tenant router forwarding provider.
  48  */
  49 public class NeutronL3Adapter {
  50
  51     /**
  52      * Logger instance.
  53      */
  54     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  55
  56     // The implementation for each of these services is resolved by the OSGi Service Manager
  57     private volatile ConfigurationService configurationService;
  58     private volatile TenantNetworkManager tenantNetworkManager;
  59     //private volatile OvsdbConfigurationService ovsdbConfigurationService;
  60     //private volatile OvsdbConnectionService connectionService;
  61     private volatile MdsalConsumer mdsalConsumer;
  62     private volatile INeutronNetworkCRUD neutronNetworkCache;
  63     private volatile INeutronSubnetCRUD neutronSubnetCache;
  64     private volatile INeutronPortCRUD neutronPortCache;
  65     private volatile L3ForwardingProvider l3ForwardingProvider;
  66     private volatile InboundNatProvider inboundNatProvider;
  67     private volatile OutboundNatProvider outboundNatProvider;
  68     private volatile ArpProvider arpProvider;
  69     private volatile RoutingProvider routingProvider;
  70
  71     private Set<String> inboundIpRewriteCache;
  72     private Set<String> outboundIpRewriteCache;
  73     private Set<String> inboundIpRewriteExclusionCache;
  74     private Set<String> outboundIpRewriteExclusionCache;
  75     private Set<String> routerInterfacesCache;
  76     private Set<String> staticArpEntryCache;
  77     private Set<String> l3ForwardingCache;
  78     private Set<String> defaultRouteCache;
  79     private Map<String, String> networkIdToRouterMacCache;
  80     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  81     private Boolean enabled = false;
  82
  83     void init() {
  84         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  85         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  86             this.inboundIpRewriteCache = new HashSet<>();
  87             this.outboundIpRewriteCache = new HashSet<>();
  88             this.inboundIpRewriteExclusionCache = new HashSet<>();
  89             this.outboundIpRewriteExclusionCache = new HashSet<>();
  90             this.routerInterfacesCache = new HashSet<>();
  91             this.staticArpEntryCache = new HashSet<>();
  92             this.l3ForwardingCache = new HashSet<>();
  93             this.defaultRouteCache = new HashSet<>();
  94             this.networkIdToRouterMacCache = new HashMap<>();
  95             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  96
  97             this.enabled = true;
  98             logger.info("OVSDB L3 forwarding is enabled");
  99         } else {
 100             logger.debug("OVSDB L3 forwarding is disabled");
 101         }
 102     }
 103
 104     //
 105     // Callbacks from OVSDB's northbound handlers
 106     //
 107
 108     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 109         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 110         if (!this.enabled)
 111             return;
 112     }
 113
 114     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 115         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 116         if (!this.enabled)
 117             return;
 118
 119         final boolean isDelete = action == Action.DELETE;
 120
 121         // Treat the port event as a router interface event if the port belongs to router. This is a
 122         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 123         //
 124         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 125             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 126                 NeutronRouter_Interface neutronRouterInterface =
 127                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 128                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 129                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 130
 131                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 132             }
 133         } else {
 134             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 135             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 136             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 137             // there.
 138             //
 139             if (!isDelete) {
 140                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 141                     NeutronRouter_Interface neutronRouterInterface =
 142                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 143                     if (neutronRouterInterface != null) {
 144                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 145                     }
 146                 }
 147             }
 148             this.updateL3ForNeutronPort(neutronPort, isDelete);
 149         }
 150     }
 151
 152     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 153         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 154         if (!this.enabled)
 155             return;
 156     }
 157
 158     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 159                                                   final NeutronRouter_Interface neutronRouterInterface,
 160                                                   Action action) {
 161         logger.debug("Router interface {} got event {}. Subnet {}",
 162                      neutronRouterInterface.getPortUUID(),
 163                      action,
 164                      neutronRouterInterface.getSubnetUUID());
 165         if (!this.enabled)
 166             return;
 167
 168         final boolean isDelete = action == Action.DELETE;
 169
 170         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 171
 172         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 173         // see if they are affected by l3
 174         //
 175         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 176             boolean currPortShouldBeDeleted = false;
 177             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 178             if (isDelete) {
 179                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 180                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 181                         currPortShouldBeDeleted = true;
 182                         break;
 183                     }
 184                 }
 185             }
 186             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 187         }
 188     }
 189
 190     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 191                                              Action action) {
 192         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 193                      neutronFloatingIP.getFixedIPAddress(),
 194                      neutronFloatingIP.getFloatingIPAddress(),
 195                      neutronFloatingIP.getFloatingNetworkUUID());
 196         if (!this.enabled)
 197             return;
 198
 199         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 200     }
 201
 202     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 203         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 204         if (!this.enabled)
 205             return;
 206     }
 207
 208     //
 209     // Callbacks from OVSDB's southbound handler
 210     //
 211     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 212                                      Action action) {
 213         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 214                      action, node, intf.getName(), neutronNetwork);
 215         if (!this.enabled)
 216             return;
 217
 218         // See if there is an external uuid, so we can find the respective neutronPort
 219         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 220         if (externalIds == null) {
 221             return;
 222         }
 223         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 224         if (neutronPortId == null) {
 225             return;
 226         }
 227         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 228         if (neutronPort == null) {
 229             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 230                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 231             return;
 232         }
 233         this.handleNeutronPortEvent(neutronPort, action);
 234     }
 235
 236     //
 237     // Internal helpers
 238     //
 239     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 240
 241         final String networkUUID = neutronPort.getNetworkUUID();
 242         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 243
 244         // If there is no router interface handling the networkUUID, we are done
 245         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 246             return;
 247         }
 248
 249         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 250         // router interface are handled via handleNeutronRouterInterfaceEvent.
 251         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 252             return;
 253         }
 254
 255         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 256         final String providerSegmentationId = neutronNetwork != null ?
 257                                               neutronNetwork.getProviderSegmentationID() : null;
 258         final String tenantMac = neutronPort.getMacAddress();
 259
 260         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 261             tenantMac == null || tenantMac.isEmpty()) {
 262             return;  // done: go no further w/out all the info needed...
 263         }
 264
 265         final Action action = isDelete ? Action.DELETE : Action.ADD;
 266         /* TODO SB_MIGRATION
 267         List<Node> nodes = connectionService.getNodes();
 268         */
 269         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 270         if (nodes.isEmpty()) {
 271             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 272         }
 273         for (Node node : nodes) {
 274             final Long dpid = getDpid(node);
 275             final boolean tenantNetworkPresentInNode =
 276                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 277             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 278                 final String tenantIpStr = neutronIP.getIpAddress();
 279                 if (tenantIpStr.isEmpty()) {
 280                     continue;
 281                 }
 282
 283                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 284                 // still needed when routing to subnets non-local to node (bug 2076).
 285                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 286
 287                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 288                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 289                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 290             }
 291         }
 292     }
 293
 294     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 295                                            String macAddress, String ipStr,
 296                                            Action actionForNode) {
 297         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 298         // will look at desired action for node.
 299         //
 300         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 301         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 302
 303         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 304             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 305                          node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 306             return;
 307         }
 308         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 309             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 310                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 311             return;
 312         }
 313
 314         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 315                                                        macAddress, ipStr, actionForNode);
 316         if (status.isSuccess()) {
 317             // Update cache
 318             if (actionForNode == Action.ADD) {
 319                 l3ForwardingCache.add(cacheKey);
 320             } else {
 321                 l3ForwardingCache.remove(cacheKey);
 322             }
 323         }
 324     }
 325
 326     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 327                                              String macAddress,
 328                                              String address,
 329                                              Action actionForNode) {
 330         Status status;
 331         try {
 332             InetAddress inetAddress = InetAddress.getByName(address);
 333             status = l3ForwardingProvider == null ?
 334                      new Status(StatusCode.SUCCESS) :
 335                      l3ForwardingProvider.programForwardingTableEntry(node, dpid, providerSegmentationId,
 336                                                                       inetAddress, macAddress, actionForNode);
 337         } catch (UnknownHostException e) {
 338             status = new Status(StatusCode.BADREQUEST);
 339         }
 340
 341         if (status.isSuccess()) {
 342             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 343                          l3ForwardingProvider == null ? "skipped" : "programmed",
 344                          macAddress, address, node, actionForNode);
 345         } else {
 346             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 347                          macAddress, address, node, actionForNode, status);
 348         }
 349         return status;
 350     }
 351
 352     // --
 353
 354     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 355                                                        Boolean isDelete) {
 356         Preconditions.checkNotNull(destNeutronRouterInterface);
 357
 358         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 359         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 360         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 361         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 362         final NeutronNetwork neutronNetwork = subnet != null ?
 363                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 364         final String destinationSegmentationId = neutronNetwork != null ?
 365                                                  neutronNetwork.getProviderSegmentationID() : null;
 366         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 367         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 368         final String cidr = subnet != null ? subnet.getCidr() : null;
 369         final int mask = getMaskLenFromCidr(cidr);
 370
 371         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 372                      destNeutronRouterInterface, isDelete);
 373
 374         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 375             cidr == null || cidr.isEmpty() ||
 376             macAddress == null || macAddress.isEmpty() ||
 377             ipList == null || ipList.isEmpty()) {
 378             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 379                          destinationSegmentationId, cidr, macAddress, ipList);
 380             return;  // done: go no further w/out all the info needed...
 381         }
 382
 383         final Action action = isDelete ? Action.DELETE : Action.ADD;
 384
 385         // Keep cache for finding router's mac from network uuid -- add
 386         //
 387         if (! isDelete) {
 388             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 389             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 390         }
 391
 392         /* TODO SB_MIGRATION
 393         List<Node> nodes = connectionService.getNodes();*/
 394         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 395         if (nodes.isEmpty()) {
 396             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 397         }
 398         for (Node node : nodes) {
 399             final Long dpid = getDpid(node);
 400             final Action actionForNode =
 401                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 402                     action : Action.DELETE;
 403
 404             for (Neutron_IPs neutronIP : ipList) {
 405                 final String ipStr = neutronIP.getIpAddress();
 406                 if (ipStr.isEmpty()) {
 407                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 408                             node.getId().getValue(), ipStr);
 409                     continue;
 410                 }
 411
 412                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 413                 //
 414                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 415                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 416                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 417                                                               neutronNetwork, destinationSegmentationId,
 418                                                               macAddress, ipStr, mask, actionForNode,
 419                                                               true /*isReflexsive*/);
 420                 }
 421
 422                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 423             }
 424
 425             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 426             // for the external neutron networks.
 427             //
 428             {
 429                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 430                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 431                                                 cidr, actionForRewriteExclusion);
 432                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 433                                                 cidr, actionForRewriteExclusion);
 434             }
 435
 436             // Default route. For non-external subnet, make sure that there is none configured.
 437             //
 438             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 439                 final Action actionForNodeDefaultRoute =
 440                         isExternal ? actionForNode : Action.DELETE;
 441                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 442                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 443                                           actionForNodeDefaultRoute);
 444             }
 445         }
 446
 447         // Keep cache for finding router's mac from network uuid -- remove
 448         //
 449         if (isDelete) {
 450             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 451             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 452         }
 453     }
 454
 455     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 456                                                            final Long dpid,
 457                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 458                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 459                                                            final NeutronNetwork dstNeutronNetwork,
 460                                                            final String destinationSegmentationId,
 461                                                            final String dstMacAddress,
 462                                                            final String destIpStr,
 463                                                            final int destMask,
 464                                                            final Action actionForNode,
 465                                                            Boolean isReflexsive) {
 466         Preconditions.checkNotNull(srcNeutronRouterInterface);
 467         Preconditions.checkNotNull(dstNeutronRouterInterface);
 468
 469         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 470         if (sourceSubnetId == null) {
 471             logger.error("Could not get provider Subnet ID from router interface {}",
 472                          srcNeutronRouterInterface.getID());
 473             return;
 474         }
 475
 476         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 477         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 478         if (sourceNetworkId == null) {
 479             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 480             return;
 481         }
 482
 483         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 484         if (sourceNetwork == null) {
 485             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 486             return;
 487         }
 488
 489         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 490             // Isolate subnets from different tenants within the same router
 491             return;
 492         }
 493         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 494         if (sourceSegmentationId == null) {
 495             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 496             return;
 497         }
 498         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 499             // Skip 'self'
 500             return;
 501         }
 502
 503         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 504                                      dstMacAddress, destIpStr, destMask, actionForNode);
 505
 506         // Flip roles src->dst; dst->src
 507         if (isReflexsive) {
 508             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 509             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 510             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 511             final String cidr2 = sourceSubnet.getCidr();
 512             final int mask2 = getMaskLenFromCidr(cidr2);
 513
 514             if (cidr2 == null || cidr2.isEmpty() ||
 515                 macAddress2 == null || macAddress2.isEmpty() ||
 516                 ipList2 == null || ipList2.isEmpty()) {
 517                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 518                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 519                 return;  // done: go no further w/out all the info needed...
 520             }
 521
 522             for (Neutron_IPs neutronIP2 : ipList2) {
 523                 final String ipStr2 = neutronIP2.getIpAddress();
 524                 if (ipStr2.isEmpty()) {
 525                     continue;
 526                 }
 527                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 528                                                           srcNeutronRouterInterface,
 529                                                           sourceNetwork, sourceSegmentationId,
 530                                                           macAddress2, ipStr2, mask2, actionForNode,
 531                                                           false /*isReflexsive*/);
 532             }
 533         }
 534     }
 535
 536     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 537                                               String destinationSegmentationId,
 538                                               String macAddress, String ipStr, int mask,
 539                                               Action actionForNode) {
 540         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 541         // will look at desired action for node.
 542         //
 543         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 544                                 ipStr + "/" + Integer.toString(mask);
 545         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 546
 547         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 548             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 549                          "action {} is already done",
 550                          node.getId().getValue(), sourceSegmentationId, destinationSegmentationId,
 551                          ipStr, mask, actionForNode);
 552             return;
 553         }
 554         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 555             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 556                          "action {} is already done",
 557                          node.getId().getValue(), sourceSegmentationId, destinationSegmentationId,
 558                          ipStr, mask, actionForNode);
 559             return;
 560         }
 561
 562         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 563                                                           macAddress, ipStr, mask, actionForNode);
 564         if (status.isSuccess()) {
 565             // Update cache
 566             if (actionForNode == Action.ADD) {
 567                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 568                 routerInterfacesCache.add(cacheKey);
 569             } else {
 570                 // TODO: multiTenantAwareRouter.removeInterface(...);
 571                 routerInterfacesCache.remove(cacheKey);
 572             }
 573         }
 574     }
 575
 576     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 577                                                 String destinationSegmentationId,
 578                                                 String macAddress,
 579                                                 String address, int mask,
 580                                                 Action actionForNode) {
 581         Status status;
 582         try {
 583             InetAddress inetAddress = InetAddress.getByName(address);
 584             status = routingProvider == null ?
 585                      new Status(StatusCode.SUCCESS) :
 586                      routingProvider.programRouterInterface(node, dpid, sourceSegmentationId, destinationSegmentationId,
 587                                                             macAddress, inetAddress, mask, actionForNode);
 588         } catch (UnknownHostException e) {
 589             status = new Status(StatusCode.BADREQUEST);
 590         }
 591
 592         if (status.isSuccess()) {
 593             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 594                          routingProvider == null ? "skipped" : "programmed",
 595                          macAddress, address, mask, node, actionForNode);
 596         } else {
 597             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 598                          macAddress, address, mask, node, actionForNode, status);
 599         }
 600         return status;
 601     }
 602
 603     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 604                                         String macAddress, String ipStr,
 605                                         Action actionForNode) {
 606         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 607         // will look at desired action for node.
 608         //
 609         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 610         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 611
 612         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 613             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 614                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 615             return;
 616         }
 617         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 618             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 619                     node.getId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 620             return;
 621         }
 622
 623         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 624                                                     macAddress, ipStr, actionForNode);
 625         if (status.isSuccess()) {
 626             // Update cache
 627             if (actionForNode == Action.ADD) {
 628                 staticArpEntryCache.add(cacheKey);
 629             } else {
 630                 staticArpEntryCache.remove(cacheKey);
 631             }
 632         }
 633     }
 634
 635     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 636                                                 String macAddress,
 637                                                 String address,
 638                                                 Action actionForNode) {
 639         Status status;
 640         try {
 641             InetAddress inetAddress = InetAddress.getByName(address);
 642             status = arpProvider == null ?
 643                      new Status(StatusCode.SUCCESS) :
 644                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 645                                                        macAddress, inetAddress, actionForNode);
 646         } catch (UnknownHostException e) {
 647             status = new Status(StatusCode.BADREQUEST);
 648         }
 649
 650         if (status.isSuccess()) {
 651             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 652                          arpProvider == null ? "skipped" : "programmed",
 653                          macAddress, address, node, actionForNode);
 654         } else {
 655             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 656                          macAddress, address, node, actionForNode, status);
 657         }
 658         return status;
 659     }
 660
 661     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 662                                                  final boolean isInbound, String cidr,
 663                                                  Action actionForRewriteExclusion) {
 664         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 665         // will look at desired action for node.
 666         //
 667         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 668         final Boolean isProgrammed = isInbound ?
 669                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 670                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 671
 672         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 673             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 674                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 675                          actionForRewriteExclusion);
 676             return;
 677         }
 678         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 679             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 680                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 681                          actionForRewriteExclusion);
 682             return;
 683         }
 684
 685         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 686                                                              isInbound, actionForRewriteExclusion);
 687         if (status.isSuccess()) {
 688             // Update cache
 689             if (actionForRewriteExclusion == Action.ADD) {
 690                 if (isInbound) {
 691                     inboundIpRewriteExclusionCache.add(cacheKey);
 692                 } else {
 693                     outboundIpRewriteExclusionCache.add(cacheKey);
 694                 }
 695             } else {
 696                 if (isInbound) {
 697                     inboundIpRewriteExclusionCache.remove(cacheKey);
 698                 } else {
 699                     outboundIpRewriteExclusionCache.remove(cacheKey);
 700                 }
 701             }
 702         }
 703     }
 704
 705     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 706                                                    final boolean isInbound, Action actionForNode) {
 707         Status status;
 708         if (isInbound) {
 709             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 710                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 711                                                                   actionForNode);
 712         } else {
 713             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 714                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 715                                                                    actionForNode);
 716         }
 717
 718         if (status.isSuccess()) {
 719             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 720             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 721                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 722                          cidr, node, actionForNode);
 723         } else {
 724             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 725                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 726         }
 727         return status;
 728     }
 729
 730     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 731                                            String defaultGatewayMacAddress, String gatewayIp,
 732                                            Action actionForNodeDefaultRoute) {
 733         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 734         // will look at desired action for node.
 735         //
 736         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 737         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 738
 739         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 740             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 741                          node.getId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 742                          actionForNodeDefaultRoute);
 743             return;
 744         }
 745         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 746             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 747                          node.getId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 748                          actionForNodeDefaultRoute);
 749             return;
 750         }
 751
 752         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 753                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 754         if (status.isSuccess()) {
 755             // Update cache
 756             if (actionForNodeDefaultRoute == Action.ADD) {
 757                 defaultRouteCache.add(cacheKey);
 758             } else {
 759                 defaultRouteCache.remove(cacheKey);
 760             }
 761         }
 762     }
 763
 764     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 765                                           String defaultGatewayMacAddress,
 766                                           String gatewayIp,
 767                                           Action actionForNodeDefaultRoute) {
 768         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 769         if (defaultGatewayMacAddress == null) {
 770             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 771                          gatewayIp, node, actionForNodeDefaultRoute);
 772             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 773         }
 774
 775         Status status;
 776         try {
 777             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 778             status = routingProvider == null ?
 779                      new Status(StatusCode.SUCCESS) :
 780                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 781                                                               defaultGatewayMacAddress, inetAddress,
 782                                                               actionForNodeDefaultRoute);
 783         } catch (UnknownHostException e) {
 784             status = new Status(StatusCode.BADREQUEST);
 785         }
 786
 787         if (status.isSuccess()) {
 788             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 789                          routingProvider == null ? "skipped" : "programmed",
 790                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 791         } else {
 792             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 793                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 794         }
 795         return status;
 796     }
 797
 798     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 799         Preconditions.checkNotNull(neutronFloatingIP);
 800
 801         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 802         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 803
 804         // If there is no router interface handling the networkUUID, we are done
 805         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 806             return;
 807         }
 808
 809         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 810         final String providerSegmentationId = neutronNetwork != null ?
 811                                               neutronNetwork.getProviderSegmentationID() : null;
 812         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 813         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 814
 815         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 816             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 817             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 818             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 819             return;  // done: go no further w/out all the info needed...
 820         }
 821
 822         final Action action = isDelete ? Action.DELETE : Action.ADD;
 823         /* TODO SB_MIGRATION
 824         List<Node> nodes = connectionService.getNodes();*/
 825         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 826         if (nodes.isEmpty()) {
 827             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 828         }
 829         for (Node node : nodes) {
 830             final Long dpid = getDpid(node);
 831             final Action actionForNode =
 832                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 833                     action : Action.DELETE;
 834
 835             // Rewrite from float to fixed and vice-versa
 836             //
 837             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 838                                    floatingIpAddress, fixedIPAddress, actionForNode);
 839             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 840                                    fixedIPAddress, floatingIpAddress, actionForNode);
 841
 842             // Respond to arps for the floating ip address
 843             //
 844             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 845                                    actionForNode);
 846         }
 847     }
 848
 849     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 850                                         final boolean isInbound,
 851                                         String matchAddress, String rewriteAddress,
 852                                         Action actionForNode) {
 853         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 854         // will look at desired action for node.
 855         //
 856         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 857                                 matchAddress + ":" + rewriteAddress;
 858         final Boolean isProgrammed = isInbound ?
 859                                      inboundIpRewriteCache.contains(cacheKey) :
 860                                      outboundIpRewriteCache.contains(cacheKey);
 861
 862         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 863             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 864                          " is already done",
 865                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 866                          matchAddress, rewriteAddress, actionForNode);
 867             return;
 868         }
 869         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 870             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 871                          " is already done",
 872                          node.getId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 873                          matchAddress, rewriteAddress, actionForNode);
 874             return;
 875         }
 876
 877         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 878                                                     matchAddress, rewriteAddress, actionForNode);
 879         if (status.isSuccess()) {
 880             // Update cache
 881             if (actionForNode == Action.ADD) {
 882                 if (isInbound) {
 883                     inboundIpRewriteCache.add(cacheKey);
 884                 } else {
 885                     outboundIpRewriteCache.add(cacheKey);
 886                 }
 887             } else {
 888                 if (isInbound) {
 889                     inboundIpRewriteCache.remove(cacheKey);
 890                 } else {
 891                     outboundIpRewriteCache.remove(cacheKey);
 892                 }
 893             }
 894         }
 895     }
 896
 897     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 898                                           final boolean isInbound,
 899                                           String matchAddress, String rewriteAddress,
 900                                           Action actionForNode) {
 901         Status status;
 902         try {
 903             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 904             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 905             if (isInbound) {
 906                 status = inboundNatProvider == null ?
 907                          new Status(StatusCode.SUCCESS) :
 908                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 909                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 910             } else {
 911                 status = outboundNatProvider == null ?
 912                          new Status(StatusCode.SUCCESS) :
 913                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 914                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 915             }
 916         } catch (UnknownHostException e) {
 917             status = new Status(StatusCode.BADREQUEST);
 918         }
 919
 920         if (status.isSuccess()) {
 921             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 922             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 923                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 924                          matchAddress, rewriteAddress, node, actionForNode);
 925         } else {
 926             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 927                          (isInbound ? "inbound" : "outbound"),
 928                          matchAddress, rewriteAddress, node, actionForNode, status);
 929         }
 930         return status;
 931     }
 932
 933     //
 934     // More Internals
 935     //
 936
 937     private int getMaskLenFromCidr(String cidr) {
 938         if (cidr == null) return 0;
 939         String[] splits = cidr.split("/");
 940         if (splits.length != 2) return 0;
 941
 942         int result;
 943         try {
 944             result = Integer.parseInt(splits[1].trim());
 945         }
 946         catch (NumberFormatException nfe)
 947         {
 948             result = 0;
 949         }
 950         return result;
 951     }
 952
 953     private Long getDpid (Node node) {
 954         return 0L;
 955         /* TODO SB_MIGRATION
 956         Preconditions.checkNotNull(ovsdbConfigurationService);
 957
 958         String bridgeName = configurationService.getIntegrationBridgeName();
 959         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 960         if (bridgeUuid == null) {
 961             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 962             return 0L;
 963         }
 964
 965         try {
 966             Row bridgeRow =  ovsdbConfigurationService
 967                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 968             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 969             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 970             if (dpids == null || dpids.size() == 0) return 0L;
 971             return StringConvertor.dpidStringToLong((String) dpids.toArray()[0]);
 972         } catch (Exception e) {
 973             logger.error("Error finding Bridge's OF DPID", e);
 974             return 0L;
 975         }*/
 976     }
 977
 978     private String getInternalBridgeUUID (Node node, String bridgeName) {
 979         /* TODO SB_MIGRATION
 980         Preconditions.checkNotNull(ovsdbConfigurationService);
 981         try {
 982             Map<String, Row> bridgeTable =
 983                     ovsdbConfigurationService.getRows(node,
 984                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 985             if (bridgeTable == null) return null;
 986             for (String key : bridgeTable.keySet()) {
 987                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 988                 if (bridge.getName().equals(bridgeName)) return key;
 989             }
 990         } catch (Exception e) {
 991             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 992         }*/
 993         return null;
 994     }
 995 }