openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import java.util.ArrayList;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.lib.notation.Row;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  27 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  28 import org.opendaylight.ovsdb.utils.config.ConfigProperties;
  29 import org.opendaylight.ovsdb.utils.mdsal.node.StringConvertor;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import org.slf4j.Logger;
  34 import org.slf4j.LoggerFactory;
  35
  36 import java.net.InetAddress;
  37 import java.net.UnknownHostException;
  38 import java.util.HashMap;
  39 import java.util.HashSet;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.Set;
  43
  44 /**
  45  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  46  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  47  * as well as the multi-tenant router forwarding provider.
  48  */
  49 public class NeutronL3Adapter {
  50
  51     /**
  52      * Logger instance.
  53      */
  54     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  55
  56     // The implementation for each of these services is resolved by the OSGi Service Manager
  57     private volatile ConfigurationService configurationService;
  58     private volatile TenantNetworkManager tenantNetworkManager;
  59     /* TODO SB_MIGRATION */
  60     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  61     private volatile OvsdbConnectionService connectionService;
  62     private volatile MdsalConsumer mdsalConsumer;
  63     private volatile INeutronNetworkCRUD neutronNetworkCache;
  64     private volatile INeutronSubnetCRUD neutronSubnetCache;
  65     private volatile INeutronPortCRUD neutronPortCache;
  66     private volatile L3ForwardingProvider l3ForwardingProvider;
  67     private volatile InboundNatProvider inboundNatProvider;
  68     private volatile OutboundNatProvider outboundNatProvider;
  69     private volatile ArpProvider arpProvider;
  70     private volatile RoutingProvider routingProvider;
  71
  72     private Set<String> inboundIpRewriteCache;
  73     private Set<String> outboundIpRewriteCache;
  74     private Set<String> inboundIpRewriteExclusionCache;
  75     private Set<String> outboundIpRewriteExclusionCache;
  76     private Set<String> routerInterfacesCache;
  77     private Set<String> staticArpEntryCache;
  78     private Set<String> l3ForwardingCache;
  79     private Set<String> defaultRouteCache;
  80     private Map<String, String> networkIdToRouterMacCache;
  81     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  82     private Boolean enabled = false;
  83
  84     void init() {
  85         final String enabledPropertyStr = ConfigProperties.getProperty(this.getClass(), "ovsdb.l3.fwd.enabled");
  86         if (enabledPropertyStr != null && enabledPropertyStr.equalsIgnoreCase("yes")) {
  87             this.inboundIpRewriteCache = new HashSet<>();
  88             this.outboundIpRewriteCache = new HashSet<>();
  89             this.inboundIpRewriteExclusionCache = new HashSet<>();
  90             this.outboundIpRewriteExclusionCache = new HashSet<>();
  91             this.routerInterfacesCache = new HashSet<>();
  92             this.staticArpEntryCache = new HashSet<>();
  93             this.l3ForwardingCache = new HashSet<>();
  94             this.defaultRouteCache = new HashSet<>();
  95             this.networkIdToRouterMacCache = new HashMap<>();
  96             this.subnetIdToRouterInterfaceCache = new HashMap<>();
  97
  98             this.enabled = true;
  99             logger.info("OVSDB L3 forwarding is enabled");
 100         } else {
 101             logger.debug("OVSDB L3 forwarding is disabled");
 102         }
 103     }
 104
 105     //
 106     // Callbacks from OVSDB's northbound handlers
 107     //
 108
 109     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 110         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 111         if (!this.enabled)
 112             return;
 113     }
 114
 115     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 116         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 117         if (!this.enabled)
 118             return;
 119
 120         final boolean isDelete = action == Action.DELETE;
 121
 122         // Treat the port event as a router interface event if the port belongs to router. This is a
 123         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 124         //
 125         if (neutronPort.getDeviceOwner().equalsIgnoreCase("network:router_interface")) {
 126             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 127                 NeutronRouter_Interface neutronRouterInterface =
 128                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 129                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 130                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 131
 132                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 133             }
 134         } else {
 135             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 136             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 137             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 138             // there.
 139             //
 140             if (!isDelete) {
 141                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 142                     NeutronRouter_Interface neutronRouterInterface =
 143                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 144                     if (neutronRouterInterface != null) {
 145                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 146                     }
 147                 }
 148             }
 149             this.updateL3ForNeutronPort(neutronPort, isDelete);
 150         }
 151     }
 152
 153     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 154         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 155         if (!this.enabled)
 156             return;
 157     }
 158
 159     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 160                                                   final NeutronRouter_Interface neutronRouterInterface,
 161                                                   Action action) {
 162         logger.debug("Router interface {} got event {}. Subnet {}",
 163                      neutronRouterInterface.getPortUUID(),
 164                      action,
 165                      neutronRouterInterface.getSubnetUUID());
 166         if (!this.enabled)
 167             return;
 168
 169         final boolean isDelete = action == Action.DELETE;
 170
 171         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 172
 173         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 174         // see if they are affected by l3
 175         //
 176         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 177             boolean currPortShouldBeDeleted = false;
 178             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 179             if (isDelete) {
 180                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 181                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 182                         currPortShouldBeDeleted = true;
 183                         break;
 184                     }
 185                 }
 186             }
 187             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 188         }
 189     }
 190
 191     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 192                                              Action action) {
 193         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 194                      neutronFloatingIP.getFixedIPAddress(),
 195                      neutronFloatingIP.getFloatingIPAddress(),
 196                      neutronFloatingIP.getFloatingNetworkUUID());
 197         if (!this.enabled)
 198             return;
 199
 200         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 201     }
 202
 203     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 204         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 205         if (!this.enabled)
 206             return;
 207     }
 208
 209     //
 210     // Callbacks from OVSDB's southbound handler
 211     //
 212     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 213                                      Action action) {
 214         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 215                      action, node, intf.getName(), neutronNetwork);
 216         if (!this.enabled)
 217             return;
 218
 219         // See if there is an external uuid, so we can find the respective neutronPort
 220         Map<String, String> externalIds = intf.getExternalIdsColumn().getData();
 221         if (externalIds == null) {
 222             return;
 223         }
 224         String neutronPortId = externalIds.get(Constants.EXTERNAL_ID_INTERFACE_ID);
 225         if (neutronPortId == null) {
 226             return;
 227         }
 228         final NeutronPort neutronPort = neutronPortCache.getPort(neutronPortId);
 229         if (neutronPort == null) {
 230             logger.warn("southbound interface {} node:{} interface:{}, neutronNetwork:{} did not find port:{}",
 231                         action, node, intf.getName(), neutronNetwork, neutronPortId);
 232             return;
 233         }
 234         this.handleNeutronPortEvent(neutronPort, action);
 235     }
 236
 237     //
 238     // Internal helpers
 239     //
 240     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 241
 242         final String networkUUID = neutronPort.getNetworkUUID();
 243         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 244
 245         // If there is no router interface handling the networkUUID, we are done
 246         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 247             return;
 248         }
 249
 250         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 251         // router interface are handled via handleNeutronRouterInterfaceEvent.
 252         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 253             return;
 254         }
 255
 256         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 257         final String providerSegmentationId = neutronNetwork != null ?
 258                                               neutronNetwork.getProviderSegmentationID() : null;
 259         final String tenantMac = neutronPort.getMacAddress();
 260
 261         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 262             tenantMac == null || tenantMac.isEmpty()) {
 263             return;  // done: go no further w/out all the info needed...
 264         }
 265
 266         final Action action = isDelete ? Action.DELETE : Action.ADD;
 267         /* TODO SB_MIGRATION
 268         List<Node> nodes = connectionService.getNodes();
 269         */
 270         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 271         if (nodes.isEmpty()) {
 272             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 273         }
 274         for (Node node : nodes) {
 275             final Long dpid = getDpid(node);
 276             final boolean tenantNetworkPresentInNode =
 277                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 278             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 279                 final String tenantIpStr = neutronIP.getIpAddress();
 280                 if (tenantIpStr.isEmpty()) {
 281                     continue;
 282                 }
 283
 284                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 285                 // still needed when routing to subnets non-local to node (bug 2076).
 286                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 287
 288                 // Configure distributed ARP responder. Only needed if tenant network exists in node.
 289                 programStaticArpStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr,
 290                                        tenantNetworkPresentInNode ? action : Action.DELETE);
 291             }
 292         }
 293     }
 294
 295     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 296                                            String macAddress, String ipStr,
 297                                            Action actionForNode) {
 298         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 299         // will look at desired action for node.
 300         //
 301         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 302         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 303
 304         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 305             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 306                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 307             return;
 308         }
 309         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 310             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 311                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 312             return;
 313         }
 314
 315         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 316                                                        macAddress, ipStr, actionForNode);
 317         if (status.isSuccess()) {
 318             // Update cache
 319             if (actionForNode == Action.ADD) {
 320                 l3ForwardingCache.add(cacheKey);
 321             } else {
 322                 l3ForwardingCache.remove(cacheKey);
 323             }
 324         }
 325     }
 326
 327     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 328                                              String macAddress,
 329                                              String address,
 330                                              Action actionForNode) {
 331         Status status;
 332         try {
 333             InetAddress inetAddress = InetAddress.getByName(address);
 334             status = l3ForwardingProvider == null ?
 335                      new Status(StatusCode.SUCCESS) :
 336                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 337                                                                       inetAddress, macAddress, actionForNode);
 338         } catch (UnknownHostException e) {
 339             status = new Status(StatusCode.BADREQUEST);
 340         }
 341
 342         if (status.isSuccess()) {
 343             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 344                          l3ForwardingProvider == null ? "skipped" : "programmed",
 345                          macAddress, address, node, actionForNode);
 346         } else {
 347             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 348                          macAddress, address, node, actionForNode, status);
 349         }
 350         return status;
 351     }
 352
 353     // --
 354
 355     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 356                                                        Boolean isDelete) {
 357         Preconditions.checkNotNull(destNeutronRouterInterface);
 358
 359         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 360         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 361         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 362         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 363         final NeutronNetwork neutronNetwork = subnet != null ?
 364                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 365         final String destinationSegmentationId = neutronNetwork != null ?
 366                                                  neutronNetwork.getProviderSegmentationID() : null;
 367         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 368         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 369         final String cidr = subnet != null ? subnet.getCidr() : null;
 370         final int mask = getMaskLenFromCidr(cidr);
 371
 372         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 373                      destNeutronRouterInterface, isDelete);
 374
 375         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 376             cidr == null || cidr.isEmpty() ||
 377             macAddress == null || macAddress.isEmpty() ||
 378             ipList == null || ipList.isEmpty()) {
 379             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 380                          destinationSegmentationId, cidr, macAddress, ipList);
 381             return;  // done: go no further w/out all the info needed...
 382         }
 383
 384         final Action action = isDelete ? Action.DELETE : Action.ADD;
 385
 386         // Keep cache for finding router's mac from network uuid -- add
 387         //
 388         if (! isDelete) {
 389             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 390             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 391         }
 392
 393         /* TODO SB_MIGRATION
 394         List<Node> nodes = connectionService.getNodes();*/
 395         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 396         if (nodes.isEmpty()) {
 397             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 398         }
 399         for (Node node : nodes) {
 400             final Long dpid = getDpid(node);
 401             final Action actionForNode =
 402                     tenantNetworkManager.isTenantNetworkPresentInNode(node, destinationSegmentationId) ?
 403                     action : Action.DELETE;
 404
 405             for (Neutron_IPs neutronIP : ipList) {
 406                 final String ipStr = neutronIP.getIpAddress();
 407                 if (ipStr.isEmpty()) {
 408                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 409                             node.getNodeId().getValue(), ipStr);
 410                     continue;
 411                 }
 412
 413                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 414                 //
 415                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 416                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 417                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 418                                                               neutronNetwork, destinationSegmentationId,
 419                                                               macAddress, ipStr, mask, actionForNode,
 420                                                               true /*isReflexsive*/);
 421                 }
 422
 423                 programStaticArpStage1(node, dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 424             }
 425
 426             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 427             // for the external neutron networks.
 428             //
 429             {
 430                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 431                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, true /* isInbound */,
 432                                                 cidr, actionForRewriteExclusion);
 433                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, false /* isInbound */,
 434                                                 cidr, actionForRewriteExclusion);
 435             }
 436
 437             // Default route. For non-external subnet, make sure that there is none configured.
 438             //
 439             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 440                 final Action actionForNodeDefaultRoute =
 441                         isExternal ? actionForNode : Action.DELETE;
 442                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 443                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 444                                           actionForNodeDefaultRoute);
 445             }
 446         }
 447
 448         // Keep cache for finding router's mac from network uuid -- remove
 449         //
 450         if (isDelete) {
 451             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 452             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 453         }
 454     }
 455
 456     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 457                                                            final Long dpid,
 458                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 459                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 460                                                            final NeutronNetwork dstNeutronNetwork,
 461                                                            final String destinationSegmentationId,
 462                                                            final String dstMacAddress,
 463                                                            final String destIpStr,
 464                                                            final int destMask,
 465                                                            final Action actionForNode,
 466                                                            Boolean isReflexsive) {
 467         Preconditions.checkNotNull(srcNeutronRouterInterface);
 468         Preconditions.checkNotNull(dstNeutronRouterInterface);
 469
 470         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 471         if (sourceSubnetId == null) {
 472             logger.error("Could not get provider Subnet ID from router interface {}",
 473                          srcNeutronRouterInterface.getID());
 474             return;
 475         }
 476
 477         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 478         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 479         if (sourceNetworkId == null) {
 480             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 481             return;
 482         }
 483
 484         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 485         if (sourceNetwork == null) {
 486             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 487             return;
 488         }
 489
 490         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 491             // Isolate subnets from different tenants within the same router
 492             return;
 493         }
 494         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 495         if (sourceSegmentationId == null) {
 496             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 497             return;
 498         }
 499         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 500             // Skip 'self'
 501             return;
 502         }
 503
 504         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 505                                      dstMacAddress, destIpStr, destMask, actionForNode);
 506
 507         // Flip roles src->dst; dst->src
 508         if (isReflexsive) {
 509             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 510             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 511             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 512             final String cidr2 = sourceSubnet.getCidr();
 513             final int mask2 = getMaskLenFromCidr(cidr2);
 514
 515             if (cidr2 == null || cidr2.isEmpty() ||
 516                 macAddress2 == null || macAddress2.isEmpty() ||
 517                 ipList2 == null || ipList2.isEmpty()) {
 518                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 519                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 520                 return;  // done: go no further w/out all the info needed...
 521             }
 522
 523             for (Neutron_IPs neutronIP2 : ipList2) {
 524                 final String ipStr2 = neutronIP2.getIpAddress();
 525                 if (ipStr2.isEmpty()) {
 526                     continue;
 527                 }
 528                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 529                                                           srcNeutronRouterInterface,
 530                                                           sourceNetwork, sourceSegmentationId,
 531                                                           macAddress2, ipStr2, mask2, actionForNode,
 532                                                           false /*isReflexsive*/);
 533             }
 534         }
 535     }
 536
 537     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 538                                               String destinationSegmentationId,
 539                                               String macAddress, String ipStr, int mask,
 540                                               Action actionForNode) {
 541         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 542         // will look at desired action for node.
 543         //
 544         final String cacheKey = node.toString() + ":" + sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 545                                 ipStr + "/" + Integer.toString(mask);
 546         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 547
 548         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 549             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 550                          "action {} is already done",
 551                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 552                          ipStr, mask, actionForNode);
 553             return;
 554         }
 555         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 556             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 557                          "action {} is already done",
 558                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 559                          ipStr, mask, actionForNode);
 560             return;
 561         }
 562
 563         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 564                                                           macAddress, ipStr, mask, actionForNode);
 565         if (status.isSuccess()) {
 566             // Update cache
 567             if (actionForNode == Action.ADD) {
 568                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 569                 routerInterfacesCache.add(cacheKey);
 570             } else {
 571                 // TODO: multiTenantAwareRouter.removeInterface(...);
 572                 routerInterfacesCache.remove(cacheKey);
 573             }
 574         }
 575     }
 576
 577     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 578                                                 String destinationSegmentationId,
 579                                                 String macAddress,
 580                                                 String address, int mask,
 581                                                 Action actionForNode) {
 582         Status status;
 583         try {
 584             InetAddress inetAddress = InetAddress.getByName(address);
 585             status = routingProvider == null ?
 586                      new Status(StatusCode.SUCCESS) :
 587                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 588                                                             macAddress, inetAddress, mask, actionForNode);
 589         } catch (UnknownHostException e) {
 590             status = new Status(StatusCode.BADREQUEST);
 591         }
 592
 593         if (status.isSuccess()) {
 594             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 595                          routingProvider == null ? "skipped" : "programmed",
 596                          macAddress, address, mask, node, actionForNode);
 597         } else {
 598             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 599                          macAddress, address, mask, node, actionForNode, status);
 600         }
 601         return status;
 602     }
 603
 604     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 605                                         String macAddress, String ipStr,
 606                                         Action actionForNode) {
 607         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 608         // will look at desired action for node.
 609         //
 610         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 611         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 612
 613         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 614             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 615                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 616             return;
 617         }
 618         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 619             logger.trace("programStaticArpStage1 node {} providerId {} mac {} ip {} action {} is already done",
 620                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 621             return;
 622         }
 623
 624         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 625                                                     macAddress, ipStr, actionForNode);
 626         if (status.isSuccess()) {
 627             // Update cache
 628             if (actionForNode == Action.ADD) {
 629                 staticArpEntryCache.add(cacheKey);
 630             } else {
 631                 staticArpEntryCache.remove(cacheKey);
 632             }
 633         }
 634     }
 635
 636     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 637                                                 String macAddress,
 638                                                 String address,
 639                                                 Action actionForNode) {
 640         Status status;
 641         try {
 642             InetAddress inetAddress = InetAddress.getByName(address);
 643             status = arpProvider == null ?
 644                      new Status(StatusCode.SUCCESS) :
 645                      arpProvider.programStaticArpEntry(dpid, providerSegmentationId,
 646                                                        macAddress, inetAddress, actionForNode);
 647         } catch (UnknownHostException e) {
 648             status = new Status(StatusCode.BADREQUEST);
 649         }
 650
 651         if (status.isSuccess()) {
 652             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 653                          arpProvider == null ? "skipped" : "programmed",
 654                          macAddress, address, node, actionForNode);
 655         } else {
 656             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 657                          macAddress, address, node, actionForNode, status);
 658         }
 659         return status;
 660     }
 661
 662     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 663                                                  final boolean isInbound, String cidr,
 664                                                  Action actionForRewriteExclusion) {
 665         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 666         // will look at desired action for node.
 667         //
 668         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 669         final Boolean isProgrammed = isInbound ?
 670                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 671                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 672
 673         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 674             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 675                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 676                          actionForRewriteExclusion);
 677             return;
 678         }
 679         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 680             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} {} cidr {} action {} is already done",
 681                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound" : "outbound", cidr,
 682                          actionForRewriteExclusion);
 683             return;
 684         }
 685
 686         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 687                                                              isInbound, actionForRewriteExclusion);
 688         if (status.isSuccess()) {
 689             // Update cache
 690             if (actionForRewriteExclusion == Action.ADD) {
 691                 if (isInbound) {
 692                     inboundIpRewriteExclusionCache.add(cacheKey);
 693                 } else {
 694                     outboundIpRewriteExclusionCache.add(cacheKey);
 695                 }
 696             } else {
 697                 if (isInbound) {
 698                     inboundIpRewriteExclusionCache.remove(cacheKey);
 699                 } else {
 700                     outboundIpRewriteExclusionCache.remove(cacheKey);
 701                 }
 702             }
 703         }
 704     }
 705
 706     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 707                                                    final boolean isInbound, Action actionForNode) {
 708         Status status;
 709         if (isInbound) {
 710             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 711                      inboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 712                                                                   actionForNode);
 713         } else {
 714             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 715                      outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr,
 716                                                                    actionForNode);
 717         }
 718
 719         if (status.isSuccess()) {
 720             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 721             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 722                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 723                          cidr, node, actionForNode);
 724         } else {
 725             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 726                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 727         }
 728         return status;
 729     }
 730
 731     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 732                                            String defaultGatewayMacAddress, String gatewayIp,
 733                                            Action actionForNodeDefaultRoute) {
 734         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 735         // will look at desired action for node.
 736         //
 737         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 738         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 739
 740         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
 741             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 742                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 743                          actionForNodeDefaultRoute);
 744             return;
 745         }
 746         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
 747             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
 748                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 749                          actionForNodeDefaultRoute);
 750             return;
 751         }
 752
 753         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 754                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 755         if (status.isSuccess()) {
 756             // Update cache
 757             if (actionForNodeDefaultRoute == Action.ADD) {
 758                 defaultRouteCache.add(cacheKey);
 759             } else {
 760                 defaultRouteCache.remove(cacheKey);
 761             }
 762         }
 763     }
 764
 765     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 766                                           String defaultGatewayMacAddress,
 767                                           String gatewayIp,
 768                                           Action actionForNodeDefaultRoute) {
 769         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
 770         if (defaultGatewayMacAddress == null) {
 771             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
 772                          gatewayIp, node, actionForNodeDefaultRoute);
 773             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
 774         }
 775
 776         Status status;
 777         try {
 778             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 779             status = routingProvider == null ?
 780                      new Status(StatusCode.SUCCESS) :
 781                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
 782                                                               defaultGatewayMacAddress, inetAddress,
 783                                                               actionForNodeDefaultRoute);
 784         } catch (UnknownHostException e) {
 785             status = new Status(StatusCode.BADREQUEST);
 786         }
 787
 788         if (status.isSuccess()) {
 789             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 790                          routingProvider == null ? "skipped" : "programmed",
 791                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 792         } else {
 793             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 794                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 795         }
 796         return status;
 797     }
 798
 799     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 800         Preconditions.checkNotNull(neutronFloatingIP);
 801
 802         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 803         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 804
 805         // If there is no router interface handling the networkUUID, we are done
 806         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 807             return;
 808         }
 809
 810         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 811         final String providerSegmentationId = neutronNetwork != null ?
 812                                               neutronNetwork.getProviderSegmentationID() : null;
 813         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 814         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 815
 816         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 817             // routerMacAddress == null || routerMacAddress.isEmpty() ||
 818             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 819             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 820             return;  // done: go no further w/out all the info needed...
 821         }
 822
 823         final Action action = isDelete ? Action.DELETE : Action.ADD;
 824         /* TODO SB_MIGRATION
 825         List<Node> nodes = connectionService.getNodes();*/
 826         List<Node> nodes = new ArrayList<>(); // TODO SB_MIGRATION
 827         if (nodes.isEmpty()) {
 828             logger.trace("programFlowsForFloatingIP has no nodes to work with");
 829         }
 830         for (Node node : nodes) {
 831             final Long dpid = getDpid(node);
 832             final Action actionForNode =
 833                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 834                     action : Action.DELETE;
 835
 836             // Rewrite from float to fixed and vice-versa
 837             //
 838             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 839                                    floatingIpAddress, fixedIPAddress, actionForNode);
 840             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 841                                    fixedIPAddress, floatingIpAddress, actionForNode);
 842
 843             // Respond to arps for the floating ip address
 844             //
 845             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 846                                    actionForNode);
 847         }
 848     }
 849
 850     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 851                                         final boolean isInbound,
 852                                         String matchAddress, String rewriteAddress,
 853                                         Action actionForNode) {
 854         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 855         // will look at desired action for node.
 856         //
 857         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 858                                 matchAddress + ":" + rewriteAddress;
 859         final Boolean isProgrammed = isInbound ?
 860                                      inboundIpRewriteCache.contains(cacheKey) :
 861                                      outboundIpRewriteCache.contains(cacheKey);
 862
 863         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 864             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 865                          " is already done",
 866                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 867                          matchAddress, rewriteAddress, actionForNode);
 868             return;
 869         }
 870         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 871             logger.trace("programIpRewriteStage1 node {} providerId {} {} matchAddr {} rewriteAddr {} action {}" +
 872                          " is already done",
 873                          node.getNodeId().getValue(), providerSegmentationId, isInbound ? "inbound": "outbound",
 874                          matchAddress, rewriteAddress, actionForNode);
 875             return;
 876         }
 877
 878         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 879                                                     matchAddress, rewriteAddress, actionForNode);
 880         if (status.isSuccess()) {
 881             // Update cache
 882             if (actionForNode == Action.ADD) {
 883                 if (isInbound) {
 884                     inboundIpRewriteCache.add(cacheKey);
 885                 } else {
 886                     outboundIpRewriteCache.add(cacheKey);
 887                 }
 888             } else {
 889                 if (isInbound) {
 890                     inboundIpRewriteCache.remove(cacheKey);
 891                 } else {
 892                     outboundIpRewriteCache.remove(cacheKey);
 893                 }
 894             }
 895         }
 896     }
 897
 898     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 899                                           final boolean isInbound,
 900                                           String matchAddress, String rewriteAddress,
 901                                           Action actionForNode) {
 902         Status status;
 903         try {
 904             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 905             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 906             if (isInbound) {
 907                 status = inboundNatProvider == null ?
 908                          new Status(StatusCode.SUCCESS) :
 909                          inboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 910                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 911             } else {
 912                 status = outboundNatProvider == null ?
 913                          new Status(StatusCode.SUCCESS) :
 914                          outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
 915                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 916             }
 917         } catch (UnknownHostException e) {
 918             status = new Status(StatusCode.BADREQUEST);
 919         }
 920
 921         if (status.isSuccess()) {
 922             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 923             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 924                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 925                          matchAddress, rewriteAddress, node, actionForNode);
 926         } else {
 927             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 928                          (isInbound ? "inbound" : "outbound"),
 929                          matchAddress, rewriteAddress, node, actionForNode, status);
 930         }
 931         return status;
 932     }
 933
 934     //
 935     // More Internals
 936     //
 937
 938     private int getMaskLenFromCidr(String cidr) {
 939         if (cidr == null) return 0;
 940         String[] splits = cidr.split("/");
 941         if (splits.length != 2) return 0;
 942
 943         int result;
 944         try {
 945             result = Integer.parseInt(splits[1].trim());
 946         }
 947         catch (NumberFormatException nfe)
 948         {
 949             result = 0;
 950         }
 951         return result;
 952     }
 953
 954     private Long getDpid (Node node) {
 955         /* TODO SB_MIGRATION */
 956         Preconditions.checkNotNull(ovsdbConfigurationService);
 957
 958         String bridgeName = configurationService.getIntegrationBridgeName();
 959         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 960         if (bridgeUuid == null) {
 961             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 962             return 0L;
 963         }
 964
 965         try {
 966             Row bridgeRow =  ovsdbConfigurationService
 967                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 968             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 969             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 970             if (dpids == null || dpids.size() == 0) return 0L;
 971             return StringConvertor.dpidStringToLong((String) dpids.toArray()[0]);
 972         } catch (Exception e) {
 973             logger.error("Error finding Bridge's OF DPID", e);
 974             return 0L;
 975         }
 976     }
 977
 978     private String getInternalBridgeUUID (Node node, String bridgeName) {
 979         /* TODO SB_MIGRATION */
 980         Preconditions.checkNotNull(ovsdbConfigurationService);
 981         try {
 982             Map<String, Row> bridgeTable =
 983                     ovsdbConfigurationService.getRows(node,
 984                                                       ovsdbConfigurationService.getTableName(node, Bridge.class));
 985             if (bridgeTable == null) return null;
 986             for (String key : bridgeTable.keySet()) {
 987                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 988                 if (bridge.getName().equals(bridgeName)) return key;
 989             }
 990         } catch (Exception e) {
 991             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 992         }
 993         return null;
 994     }
 995 }