openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  29 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  30
  31 import com.google.common.base.Preconditions;
  32 import org.osgi.framework.BundleContext;
  33 import org.osgi.framework.ServiceReference;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.net.InetAddress;
  38 import java.net.UnknownHostException;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.ArrayList;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45
  46 /**
  47  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  48  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  49  * as well as the multi-tenant router forwarding provider.
  50  */
  51 public class NeutronL3Adapter implements ConfigInterface {
  52     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  53
  54     // The implementation for each of these services is resolved by the OSGi Service Manager
  55     private volatile ConfigurationService configurationService;
  56     private volatile TenantNetworkManager tenantNetworkManager;
  57     private volatile NodeCacheManager nodeCacheManager;
  58     private volatile INeutronNetworkCRUD neutronNetworkCache;
  59     private volatile INeutronSubnetCRUD neutronSubnetCache;
  60     private volatile INeutronPortCRUD neutronPortCache;
  61     private volatile L3ForwardingProvider l3ForwardingProvider;
  62     private volatile InboundNatProvider inboundNatProvider;
  63     private volatile OutboundNatProvider outboundNatProvider;
  64     private volatile ArpProvider arpProvider;
  65     private volatile RoutingProvider routingProvider;
  66
  67     private class FloatIpData {
  68         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  69                     final String floatingIpAddress, final String fixedIpAddress) {
  70             this.dpid = dpid;
  71             this.ofPort = ofPort;
  72             this.segId = segId;
  73             this.macAddress = macAddress;
  74             this.floatingIpAddress = floatingIpAddress;
  75             this.fixedIpAddress = fixedIpAddress;
  76         }
  77
  78         public final Long dpid;
  79         public final Long ofPort;
  80         public final String segId;
  81         public final String macAddress;
  82         public final String floatingIpAddress;
  83         public final String fixedIpAddress;
  84     }
  85
  86     private Set<String> inboundIpRewriteCache;
  87     private Set<String> outboundIpRewriteCache;
  88     private Set<String> outboundIpRewriteExclusionCache;
  89     private Set<String> routerInterfacesCache;
  90     private Set<String> staticArpEntryCache;
  91     private Set<String> l3ForwardingCache;
  92     private Set<String> defaultRouteCache;
  93     private Map<String, String> networkIdToRouterMacCache;
  94     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
  95     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  96     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
  97     private Map<String, FloatIpData> floatIpDataMapCache;
  98     private Boolean enabled = false;
  99     private Boolean flgDistributedARPEnabled = true;
 100     private Southbound southbound;
 101
 102     final static String OWNER_ROUTER_INTERFACE = "network:router_interface";
 103     final static String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 104     final static String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 105     final static String OWNER_FLOATING_IP = "network:floatingip";
 106
 107     public NeutronL3Adapter() {
 108         logger.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 109     }
 110
 111     private void initL3AdapterMembers() {
 112         Preconditions.checkNotNull(configurationService);
 113
 114         if (configurationService.isL3ForwardingEnabled()) {
 115             this.inboundIpRewriteCache = new HashSet<>();
 116             this.outboundIpRewriteCache = new HashSet<>();
 117             this.outboundIpRewriteExclusionCache = new HashSet<>();
 118             this.routerInterfacesCache = new HashSet<>();
 119             this.staticArpEntryCache = new HashSet<>();
 120             this.l3ForwardingCache = new HashSet<>();
 121             this.defaultRouteCache = new HashSet<>();
 122             this.networkIdToRouterMacCache = new HashMap<>();
 123             this.networkIdToRouterIpListCache = new HashMap<>();
 124             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 125             this.neutronPortToDpIdCache = new HashMap<>();
 126             this.floatIpDataMapCache = new HashMap<>();
 127
 128             this.enabled = true;
 129             logger.info("OVSDB L3 forwarding is enabled");
 130             if (configurationService.isDistributedArpDisabled()) {
 131                 this.flgDistributedARPEnabled = false;
 132                 logger.info("Distributed ARP responder is disabled");
 133             } else {
 134                 logger.debug("Distributed ARP responder is enabled");
 135             }
 136         } else {
 137             logger.debug("OVSDB L3 forwarding is disabled");
 138         }
 139     }
 140
 141     //
 142     // Callbacks from OVSDB's northbound handlers
 143     //
 144
 145     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 146         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 147         if (!this.enabled)
 148             return;
 149     }
 150
 151     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 152         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 153         if (!this.enabled)
 154             return;
 155
 156         final boolean isDelete = action == Action.DELETE;
 157
 158         // Treat the port event as a router interface event if the port belongs to router. This is a
 159         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 160         //
 161         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 162             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 163             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 164                 NeutronRouter_Interface neutronRouterInterface =
 165                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 166                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 167                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 168
 169                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 170             }
 171         } else {
 172             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 173             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 174             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 175             // there.
 176             //
 177             if (!isDelete) {
 178                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 179                     NeutronRouter_Interface neutronRouterInterface =
 180                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 181                     if (neutronRouterInterface != null) {
 182                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 183                     }
 184                 }
 185             }
 186             this.updateL3ForNeutronPort(neutronPort, isDelete);
 187         }
 188     }
 189
 190     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 191         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 192         if (!this.enabled)
 193             return;
 194     }
 195
 196     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 197                                                   final NeutronRouter_Interface neutronRouterInterface,
 198                                                   Action action) {
 199         logger.debug("Router interface {} got event {}. Subnet {}",
 200                      neutronRouterInterface.getPortUUID(),
 201                      action,
 202                      neutronRouterInterface.getSubnetUUID());
 203         if (!this.enabled)
 204             return;
 205
 206         final boolean isDelete = action == Action.DELETE;
 207
 208         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 209
 210         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 211         // see if they are affected by l3
 212         //
 213         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 214             boolean currPortShouldBeDeleted = false;
 215             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 216             if (isDelete) {
 217                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 218                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 219                         currPortShouldBeDeleted = true;
 220                         break;
 221                     }
 222                 }
 223             }
 224             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 225         }
 226     }
 227
 228     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 229                                              Action action) {
 230         Preconditions.checkNotNull(neutronFloatingIP);
 231
 232         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 233                 neutronFloatingIP.getFixedIPAddress(),
 234                 neutronFloatingIP.getFloatingIPAddress(),
 235                 neutronFloatingIP.getFloatingNetworkUUID());
 236         if (!this.enabled)
 237             return;
 238
 239         // Consider action to be delete if getFixedIPAddress is null
 240         //
 241         if (neutronFloatingIP.getFixedIPAddress() == null) {
 242             action = Action.DELETE;
 243         }
 244
 245         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 246
 247         if (action != Action.DELETE) {
 248             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 249
 250             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 251         } else {
 252             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 253
 254             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 255         }
 256     }
 257
 258     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 259         Preconditions.checkNotNull(neutronFloatingIP);
 260
 261         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 262         if (fid == null) {
 263             logger.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 264                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 265             return;
 266         }
 267         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 268                                       action);
 269     }
 270
 271     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 272         Preconditions.checkNotNull(neutronFloatingIP);
 273         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 274         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 275
 276         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 277             logger.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 278                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 279             return;
 280         }
 281
 282         // find bridge Node where floating ip is configured by looking up cache for its port
 283         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 284         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 285         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 286         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 287         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 288         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 289
 290         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 291         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 292                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 293         final String providerSegmentationId = tenantNeutronNetwork != null ?
 294                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 295
 296         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 297                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 298                 floatingIpMac == null || floatingIpMac.isEmpty()) {
 299             logger.trace("Floating IP {}<->{}, found no dpid for floatPort {} tenantPortUuid {} seg {} mac {}",
 300                     fixedIpAddress,
 301                     floatingIpAddress,
 302                     neutronPortForFloatIp,
 303                     neutronTenantPortUuid,
 304                     providerSegmentationId,
 305                     floatingIpMac);
 306             return;
 307         }
 308
 309         // get ofport for patch port in br-int
 310         final Long dpId = nodeIfPair.getLeft();
 311         final Long ofPort = findOFPortForExtPatch(dpId);
 312         if (ofPort == null) {
 313             logger.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 314                     dpId);
 315             return;
 316         }
 317
 318         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 319         //
 320         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 321                 Action.ADD)) {
 322             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 323                     floatingIpAddress, fixedIpAddress);
 324             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 325             logger.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 326                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 327                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 328         }
 329     }
 330
 331     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 332         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 333         if (floatIpData == null) {
 334             logger.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 335             return;
 336         }
 337
 338         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 339                 floatIpData.floatingIpAddress, Action.DELETE)) {
 340             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 341             logger.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 342                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 343         }
 344     }
 345
 346     private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 347         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 348             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 349                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 350                 return neutronPort;
 351             }
 352         }
 353         return null;
 354     }
 355
 356     private final Long findOFPortForExtPatch(Long dpId) {
 357         final String brInt = configurationService.getIntegrationBridgeName();
 358         final String brExt = configurationService.getExternalBridgeName();
 359         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 360
 361         Preconditions.checkNotNull(dpId);
 362         Preconditions.checkNotNull(portNameInt);
 363
 364         final long dpidPrimitive = dpId.longValue();
 365         for (Node node : nodeCacheManager.getBridgeNodes()) {
 366             if (dpidPrimitive == southbound.getDataPathId(node)) {
 367                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 368                         southbound.getTerminationPointOfBridge(node, portNameInt);
 369                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 370             }
 371         }
 372         return null;
 373     }
 374
 375     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 376         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 377         if (!this.enabled)
 378             return;
 379     }
 380
 381     //
 382     // Callbacks from OVSDB's southbound handler
 383     //
 384     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 385                                      final NeutronNetwork neutronNetwork, Action action) {
 386         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 387                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 388         if (!this.enabled) {
 389             return;
 390         }
 391
 392         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 393         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 394         final Uuid interfaceUuid = intf.getInterfaceUuid();
 395
 396         logger.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 397                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 398
 399         if (neutronPort != null) {
 400             final String neutronPortUuid = neutronPort.getPortUUID();
 401
 402             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 403                     dpId != null && interfaceUuid != null) {
 404                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 405             }
 406
 407             handleNeutronPortEvent(neutronPort, action);
 408         }
 409
 410         if (action == Action.DELETE && interfaceUuid != null) {
 411             handleInterfaceEventDelete(intf, dpId);
 412         }
 413     }
 414
 415     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 416         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 417         logger.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 418                 neutronPortUuid, dpId, interfaceUuid.getValue());
 419     }
 420
 421     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 422         // Remove entry from neutronPortToDpIdCache based on interface uuid
 423         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 424             final String currPortUuid = entry.getKey();
 425             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 426                 logger.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 427                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 428                 neutronPortToDpIdCache.remove(currPortUuid);
 429                 break;
 430             }
 431         }
 432     }
 433
 434     //
 435     // Internal helpers
 436     //
 437     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 438
 439         final String networkUUID = neutronPort.getNetworkUUID();
 440         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 441
 442         // If there is no router interface handling the networkUUID, we are done
 443         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 444             return;
 445         }
 446
 447         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 448         // router interface are handled via handleNeutronRouterInterfaceEvent.
 449         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 450             return;
 451         }
 452
 453         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 454         final String providerSegmentationId = neutronNetwork != null ?
 455                                               neutronNetwork.getProviderSegmentationID() : null;
 456         final String tenantMac = neutronPort.getMacAddress();
 457
 458         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 459             tenantMac == null || tenantMac.isEmpty()) {
 460             return;  // done: go no further w/out all the info needed...
 461         }
 462
 463         final Action action = isDelete ? Action.DELETE : Action.ADD;
 464         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 465         if (nodes.isEmpty()) {
 466             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 467         }
 468         for (Node node : nodes) {
 469             final Long dpid = getDpidForIntegrationBridge(node);
 470             if (dpid == null) {
 471                 continue;
 472             }
 473
 474             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 475                 final String tenantIpStr = neutronIP.getIpAddress();
 476                 if (tenantIpStr.isEmpty()) {
 477                     continue;
 478                 }
 479
 480                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 481                 // still needed when routing to subnets non-local to node (bug 2076).
 482                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 483
 484                 // Configure distributed ARP responder
 485                 if (true == flgDistributedARPEnabled) {
 486                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 487                 }
 488             }
 489         }
 490     }
 491
 492     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 493                                            String macAddress, String ipStr,
 494                                            Action actionForNode) {
 495         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 496         // will look at desired action for node.
 497         //
 498         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 499         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 500
 501         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 502             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 503                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 504             return;
 505         }
 506         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 507             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 508                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 509             return;
 510         }
 511
 512         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 513                                                        macAddress, ipStr, actionForNode);
 514         if (status.isSuccess()) {
 515             // Update cache
 516             if (actionForNode == Action.ADD) {
 517                 l3ForwardingCache.add(cacheKey);
 518             } else {
 519                 l3ForwardingCache.remove(cacheKey);
 520             }
 521         }
 522     }
 523
 524     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 525                                              String macAddress,
 526                                              String address,
 527                                              Action actionForNode) {
 528         Status status;
 529         try {
 530             InetAddress inetAddress = InetAddress.getByName(address);
 531             status = l3ForwardingProvider == null ?
 532                      new Status(StatusCode.SUCCESS) :
 533                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 534                                                                       inetAddress, macAddress, actionForNode);
 535         } catch (UnknownHostException e) {
 536             status = new Status(StatusCode.BADREQUEST);
 537         }
 538
 539         if (status.isSuccess()) {
 540             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 541                          l3ForwardingProvider == null ? "skipped" : "programmed",
 542                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 543         } else {
 544             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 545                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 546         }
 547         return status;
 548     }
 549
 550     // --
 551
 552     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 553                                                        Boolean isDelete) {
 554         Preconditions.checkNotNull(destNeutronRouterInterface);
 555
 556         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 557         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 558         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 559         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 560         final NeutronNetwork neutronNetwork = subnet != null ?
 561                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 562         final String destinationSegmentationId = neutronNetwork != null ?
 563                                                  neutronNetwork.getProviderSegmentationID() : null;
 564         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 565         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 566         final String cidr = subnet != null ? subnet.getCidr() : null;
 567         final int mask = getMaskLenFromCidr(cidr);
 568
 569         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 570                      destNeutronRouterInterface, isDelete);
 571
 572         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 573         // the local cache
 574         if (neutronNetwork != null) {
 575             if (macAddress == null || macAddress.isEmpty()) {
 576                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 577             }
 578             if (ipList == null || ipList.isEmpty()) {
 579                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 580             }
 581         }
 582
 583         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 584             cidr == null || cidr.isEmpty() ||
 585             macAddress == null || macAddress.isEmpty() ||
 586             ipList == null || ipList.isEmpty()) {
 587             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 588                          destinationSegmentationId, cidr, macAddress, ipList);
 589             return;  // done: go no further w/out all the info needed...
 590         }
 591
 592         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 593
 594         // Keep cache for finding router's mac from network uuid -- add
 595         //
 596         if (! isDelete) {
 597             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 598             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 599             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 600         }
 601
 602         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 603         if (nodes.isEmpty()) {
 604             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 605         }
 606         for (Node node : nodes) {
 607             final Long dpid = getDpidForIntegrationBridge(node);
 608             if (dpid == null) {
 609                 continue;
 610             }
 611
 612             for (Neutron_IPs neutronIP : ipList) {
 613                 final String ipStr = neutronIP.getIpAddress();
 614                 if (ipStr.isEmpty()) {
 615                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 616                             node.getNodeId().getValue(), ipStr);
 617                     continue;
 618                 }
 619
 620                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 621                 //
 622                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 623                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 624                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 625                                                               neutronNetwork, destinationSegmentationId,
 626                                                               macAddress, ipStr, mask, actionForNode,
 627                                                               true /*isReflexsive*/);
 628                 }
 629
 630                 if (! isExternal) {
 631                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 632                             actionForNode);
 633                 }
 634                 // Enable ARP responder by default, because router interface needs to be responded always.
 635                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 636             }
 637
 638             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 639             // for the external neutron networks.
 640             //
 641             {
 642                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 643                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 644             }
 645
 646             // Default route. For non-external subnet, make sure that there is none configured.
 647             //
 648             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 649                 final Action actionForNodeDefaultRoute =
 650                         isExternal ? actionForNode : Action.DELETE;
 651                 final String defaultGatewayMacAddress = configurationService.getDefaultGatewayMacAddress(node);
 652                 programDefaultRouteStage1(node, dpid, destinationSegmentationId, defaultGatewayMacAddress, gatewayIp,
 653                                           actionForNodeDefaultRoute);
 654             }
 655         }
 656
 657         // Keep cache for finding router's mac from network uuid -- remove
 658         //
 659         if (isDelete) {
 660             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 661             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 662             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 663         }
 664     }
 665
 666     private void programFlowForNetworkFromExternal(final Node node,
 667                                                    final Long dpid,
 668                                                    final String destinationSegmentationId,
 669                                                    final String dstMacAddress,
 670                                                    final String destIpStr,
 671                                                    final int destMask,
 672                                                    final Action actionForNode) {
 673         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 674                 dstMacAddress, destIpStr, destMask, actionForNode);
 675     }
 676
 677     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 678                                                            final Long dpid,
 679                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 680                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 681                                                            final NeutronNetwork dstNeutronNetwork,
 682                                                            final String destinationSegmentationId,
 683                                                            final String dstMacAddress,
 684                                                            final String destIpStr,
 685                                                            final int destMask,
 686                                                            final Action actionForNode,
 687                                                            Boolean isReflexsive) {
 688         Preconditions.checkNotNull(srcNeutronRouterInterface);
 689         Preconditions.checkNotNull(dstNeutronRouterInterface);
 690
 691         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 692         if (sourceSubnetId == null) {
 693             logger.error("Could not get provider Subnet ID from router interface {}",
 694                          srcNeutronRouterInterface.getID());
 695             return;
 696         }
 697
 698         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 699         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 700         if (sourceNetworkId == null) {
 701             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 702             return;
 703         }
 704
 705         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 706         if (sourceNetwork == null) {
 707             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 708             return;
 709         }
 710
 711         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 712             // Isolate subnets from different tenants within the same router
 713             return;
 714         }
 715         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 716         if (sourceSegmentationId == null) {
 717             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 718             return;
 719         }
 720         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 721             // Skip 'self'
 722             return;
 723         }
 724
 725         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 726                                      dstMacAddress, destIpStr, destMask, actionForNode);
 727
 728         // Flip roles src->dst; dst->src
 729         if (isReflexsive) {
 730             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 731             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 732             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 733             final String cidr2 = sourceSubnet.getCidr();
 734             final int mask2 = getMaskLenFromCidr(cidr2);
 735
 736             if (cidr2 == null || cidr2.isEmpty() ||
 737                 macAddress2 == null || macAddress2.isEmpty() ||
 738                 ipList2 == null || ipList2.isEmpty()) {
 739                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 740                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 741                 return;  // done: go no further w/out all the info needed...
 742             }
 743
 744             for (Neutron_IPs neutronIP2 : ipList2) {
 745                 final String ipStr2 = neutronIP2.getIpAddress();
 746                 if (ipStr2.isEmpty()) {
 747                     continue;
 748                 }
 749                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 750                                                           srcNeutronRouterInterface,
 751                                                           sourceNetwork, sourceSegmentationId,
 752                                                           macAddress2, ipStr2, mask2, actionForNode,
 753                                                           false /*isReflexsive*/);
 754             }
 755         }
 756     }
 757
 758     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 759                                               String destinationSegmentationId,
 760                                               String macAddress, String ipStr, int mask,
 761                                               Action actionForNode) {
 762         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 763         // will look at desired action for node.
 764         //
 765         final String cacheKey = node.getNodeId().getValue() + ":" +
 766                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 767                                 ipStr + "/" + Integer.toString(mask);
 768         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 769
 770         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 771             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 772                          " action {} is already done",
 773                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 774                          macAddress, ipStr, mask, actionForNode);
 775             return;
 776         }
 777         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 778             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 779                          " action {} is already done",
 780                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 781                          macAddress, ipStr, mask, actionForNode);
 782             return;
 783         }
 784
 785         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 786                                                           macAddress, ipStr, mask, actionForNode);
 787         if (status.isSuccess()) {
 788             // Update cache
 789             if (actionForNode == Action.ADD) {
 790                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 791                 routerInterfacesCache.add(cacheKey);
 792             } else {
 793                 // TODO: multiTenantAwareRouter.removeInterface(...);
 794                 routerInterfacesCache.remove(cacheKey);
 795             }
 796         }
 797     }
 798
 799     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 800                                                 String destinationSegmentationId,
 801                                                 String macAddress,
 802                                                 String address, int mask,
 803                                                 Action actionForNode) {
 804         Status status;
 805         try {
 806             InetAddress inetAddress = InetAddress.getByName(address);
 807             status = routingProvider == null ?
 808                      new Status(StatusCode.SUCCESS) :
 809                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 810                                                             macAddress, inetAddress, mask, actionForNode);
 811         } catch (UnknownHostException e) {
 812             status = new Status(StatusCode.BADREQUEST);
 813         }
 814
 815         if (status.isSuccess()) {
 816             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 817                          routingProvider == null ? "skipped" : "programmed",
 818                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 819                          actionForNode);
 820         } else {
 821             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 822                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 823                          actionForNode, status);
 824         }
 825         return status;
 826     }
 827
 828     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 829                                            String macAddress, String ipStr,
 830                                            Action action) {
 831         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 832         // will look at desired action for node.
 833         //
 834         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 835         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 836
 837         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 838             logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 839                     dpid, segOrOfPort, macAddress, ipStr, action);
 840             return true;
 841         }
 842         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 843             logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 844                     dpid, segOrOfPort, macAddress, ipStr, action);
 845             return true;
 846         }
 847
 848         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 849         if (status.isSuccess()) {
 850             // Update cache
 851             if (action == Action.ADD) {
 852                 staticArpEntryCache.add(cacheKey);
 853             } else {
 854                 staticArpEntryCache.remove(cacheKey);
 855             }
 856             return true;
 857         }
 858         return false;
 859     }
 860
 861     private Status programStaticArpStage2(Long dpid,
 862                                           String segOrOfPort,
 863                                           String macAddress,
 864                                           String address,
 865                                           Action action) {
 866         Status status;
 867         try {
 868             InetAddress inetAddress = InetAddress.getByName(address);
 869             status = arpProvider == null ?
 870                      new Status(StatusCode.SUCCESS) :
 871                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
 872                                                        macAddress, inetAddress, action);
 873         } catch (UnknownHostException e) {
 874             status = new Status(StatusCode.BADREQUEST);
 875         }
 876
 877         if (status.isSuccess()) {
 878             logger.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
 879                          arpProvider == null ? "skipped" : "programmed",
 880                          macAddress, address, dpid, segOrOfPort, action);
 881         } else {
 882             logger.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
 883                          macAddress, address, dpid, segOrOfPort, action, status);
 884         }
 885         return status;
 886     }
 887
 888     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
 889                                                   String matchAddress, String rewriteAddress,
 890                                                   Action action) {
 891         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 892         // will look at desired action for node.
 893         //
 894         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
 895         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
 896
 897         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 898             logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 899                     " action {} is already done",
 900                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 901             return true;
 902         }
 903         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 904             logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 905                     " action is already done",
 906                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 907             return true;
 908         }
 909
 910         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
 911                 rewriteAddress, action);
 912         if (status.isSuccess()) {
 913             // Update cache
 914             if (action == Action.ADD) {
 915                 inboundIpRewriteCache.add(cacheKey);
 916             } else {
 917                 inboundIpRewriteCache.remove(cacheKey);
 918             }
 919             return true;
 920         }
 921         return false;
 922     }
 923
 924     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
 925                                                  String matchAddress, String rewriteAddress,
 926                                                  Action action) {
 927         Status status;
 928         try {
 929             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 930             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 931             status = inboundNatProvider == null ?
 932                     new Status(StatusCode.SUCCESS) :
 933                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
 934                             inetMatchAddress, inetRewriteAddress,
 935                             action);
 936         } catch (UnknownHostException e) {
 937             status = new Status(StatusCode.BADREQUEST);
 938         }
 939
 940         if (status.isSuccess()) {
 941             final boolean isSkipped = inboundNatProvider == null;
 942             logger.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
 943                     (isSkipped ? "skipped" : "programmed"),
 944                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 945         } else {
 946             logger.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
 947                          " status:{}",
 948                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
 949                     status);
 950         }
 951         return status;
 952     }
 953
 954     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
 955                                                  Action actionForRewriteExclusion) {
 956         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 957         // will look at desired action for node.
 958         //
 959         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
 960         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
 961
 962         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 963             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
 964                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
 965             return;
 966         }
 967         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
 968             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
 969                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
 970             return;
 971         }
 972
 973         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 974                                                              actionForRewriteExclusion);
 975         if (status.isSuccess()) {
 976             // Update cache
 977             if (actionForRewriteExclusion == Action.ADD) {
 978                     outboundIpRewriteExclusionCache.add(cacheKey);
 979             } else {
 980                     outboundIpRewriteExclusionCache.remove(cacheKey);
 981             }
 982         }
 983     }
 984
 985     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 986                                                    Action actionForNode) {
 987         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 988                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
 989
 990         if (status.isSuccess()) {
 991             final boolean isSkipped = outboundNatProvider == null;
 992             logger.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
 993                          (isSkipped ? "skipped" : "programmed"),
 994                          cidr, node.getNodeId().getValue(), actionForNode);
 995         } else {
 996             logger.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
 997                          cidr, node.getNodeId().getValue(), actionForNode, status);
 998         }
 999         return status;
1000     }
1001
1002     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
1003                                            String defaultGatewayMacAddress, String gatewayIp,
1004                                            Action actionForNodeDefaultRoute) {
1005         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1006         // will look at desired action for node.
1007         //
1008         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + gatewayIp;
1009         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
1010
1011         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE) {
1012             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
1013                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
1014                          actionForNodeDefaultRoute);
1015             return;
1016         }
1017         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE) {
1018             logger.trace("programDefaultRouteStage1 node {} providerId {} mac {} gw {} action {} is already done",
1019                          node.getNodeId().getValue(), providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
1020                          actionForNodeDefaultRoute);
1021             return;
1022         }
1023
1024         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
1025                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
1026         if (status.isSuccess()) {
1027             // Update cache
1028             if (actionForNodeDefaultRoute == Action.ADD) {
1029                 defaultRouteCache.add(cacheKey);
1030             } else {
1031                 defaultRouteCache.remove(cacheKey);
1032             }
1033         }
1034     }
1035
1036     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
1037                                           String defaultGatewayMacAddress,
1038                                           String gatewayIp,
1039                                           Action actionForNodeDefaultRoute) {
1040         // TODO: As of Helium, mac address for default gateway is required (bug 1705).
1041         if (defaultGatewayMacAddress == null) {
1042             logger.error("ProgramDefaultRoute mac not provided. gatewayIp:{} node:{} action:{}",
1043                          gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute);
1044             return new Status(StatusCode.NOTIMPLEMENTED);  // Bug 1705
1045         }
1046
1047         Status status;
1048         try {
1049             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
1050             status = routingProvider == null ?
1051                      new Status(StatusCode.SUCCESS) :
1052                      routingProvider.programDefaultRouteEntry(dpid, providerSegmentationId,
1053                                                               defaultGatewayMacAddress, inetAddress,
1054                                                               actionForNodeDefaultRoute);
1055         } catch (UnknownHostException e) {
1056             status = new Status(StatusCode.BADREQUEST);
1057         }
1058
1059         if (status.isSuccess()) {
1060             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
1061                          routingProvider == null ? "skipped" : "programmed",
1062                          defaultGatewayMacAddress, gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute);
1063         } else {
1064             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
1065                          defaultGatewayMacAddress, gatewayIp, node.getNodeId().getValue(), actionForNodeDefaultRoute, status);
1066         }
1067         return status;
1068     }
1069
1070     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
1071         Preconditions.checkNotNull(neutronFloatingIP);
1072
1073         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
1074         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
1075
1076         // If there is no router interface handling the networkUUID, we are done
1077         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
1078             return;
1079         }
1080
1081         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
1082         final String providerSegmentationId = neutronNetwork != null ?
1083                                               neutronNetwork.getProviderSegmentationID() : null;
1084         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
1085         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
1086
1087         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
1088             // routerMacAddress == null || routerMacAddress.isEmpty() ||
1089             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
1090             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
1091             return;  // done: go no further w/out all the info needed...
1092         }
1093
1094         final Action action = isDelete ? Action.DELETE : Action.ADD;
1095         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1096         if (nodes.isEmpty()) {
1097             logger.trace("programFlowsForFloatingIP has no nodes to work with");
1098         }
1099         for (Node node : nodes) {
1100             final Long dpid = getDpidForIntegrationBridge(node);
1101             if (dpid == null) {
1102                 continue;
1103             }
1104
1105             final Action actionForNode =
1106                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
1107                     action : Action.DELETE;
1108
1109             // Rewrite from float to fixed and vice-versa
1110             programIpRewriteStage1(node, dpid, providerSegmentationId, fixedIPAddress, floatingIpAddress, actionForNode);
1111         }
1112     }
1113
1114     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
1115                                         String matchAddress, String rewriteAddress,
1116                                         Action actionForNode) {
1117         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1118         // will look at desired action for node.
1119         //
1120         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" +
1121                                 matchAddress + ":" + rewriteAddress;
1122         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1123
1124         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
1125             logger.trace("programIpRewriteStage1 node {} providerId {} matchAddr {} rewriteAddr {} action {}" +
1126                          " is already done",
1127                          node.getNodeId().getValue(), providerSegmentationId, matchAddress, rewriteAddress,
1128                     actionForNode);
1129             return;
1130         }
1131         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
1132             logger.trace("programIpRewriteStage1 node {} providerId {} matchAddr {} rewriteAddr {} action {}" +
1133                          " is already done",
1134                          node.getNodeId().getValue(), providerSegmentationId, matchAddress, rewriteAddress,
1135                          actionForNode);
1136             return;
1137         }
1138
1139         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, matchAddress, rewriteAddress,
1140                                                     actionForNode);
1141         if (status.isSuccess()) {
1142             // Update cache
1143             if (actionForNode == Action.ADD) {
1144                 outboundIpRewriteCache.add(cacheKey);
1145             } else {
1146                 outboundIpRewriteCache.remove(cacheKey);
1147             }
1148         }
1149     }
1150
1151     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
1152                                           String matchAddress, String rewriteAddress,
1153                                           Action actionForNode) {
1154         Status status;
1155         try {
1156             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1157             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1158             status = outboundNatProvider == null ?
1159                     new Status(StatusCode.SUCCESS) :
1160                     outboundNatProvider.programIpRewriteRule(dpid, providerSegmentationId,
1161                                                              inetMatchAddress, inetRewriteAddress, actionForNode);
1162         } catch (UnknownHostException e) {
1163             status = new Status(StatusCode.BADREQUEST);
1164         }
1165
1166         if (status.isSuccess()) {
1167             final boolean isSkipped = outboundNatProvider == null;
1168             logger.debug("ProgramIpRewrite {} for match:{} rewrite:{} node:{} action:{}",
1169                          (isSkipped ? "skipped" : "programmed"),
1170                          matchAddress, rewriteAddress, node.getNodeId().getValue(), actionForNode);
1171         } else {
1172             logger.error("ProgramIpRewrite failed for match:{} rewrite:{} node:{} action:{} status:{}",
1173                          matchAddress, rewriteAddress, node.getNodeId().getValue(), actionForNode, status);
1174         }
1175         return status;
1176     }
1177
1178     private int getMaskLenFromCidr(String cidr) {
1179         if (cidr == null) return 0;
1180         String[] splits = cidr.split("/");
1181         if (splits.length != 2) return 0;
1182
1183         int result;
1184         try {
1185             result = Integer.parseInt(splits[1].trim());
1186         }
1187         catch (NumberFormatException nfe)
1188         {
1189             result = 0;
1190         }
1191         return result;
1192     }
1193
1194     private Long getDpidForIntegrationBridge(Node node) {
1195         // Check if node is integration bridge; and only then return its dpid
1196         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1197             return southbound.getDataPathId(node);
1198         }
1199         return null;
1200     }
1201
1202     /**
1203      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1204      *
1205      * @param ofPort the OF port number
1206      * @return the string with encoded OF port (example format "OFPort|999")
1207      */
1208     public static String encodeExcplicitOFPort(Long ofPort) {
1209         return "OFPort|" + ofPort.toString();
1210     }
1211
1212     @Override
1213     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1214         tenantNetworkManager =
1215                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1216         configurationService =
1217                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1218         arpProvider =
1219                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1220         inboundNatProvider =
1221                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1222         outboundNatProvider =
1223                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1224         routingProvider =
1225                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1226         l3ForwardingProvider =
1227                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1228         nodeCacheManager =
1229                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1230         southbound =
1231                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1232
1233         initL3AdapterMembers();
1234     }
1235
1236     @Override
1237     public void setDependencies(Object impl) {
1238         if (impl instanceof INeutronNetworkCRUD) {
1239             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1240         } else if (impl instanceof INeutronPortCRUD) {
1241             neutronPortCache = (INeutronPortCRUD)impl;
1242         } else if (impl instanceof INeutronSubnetCRUD) {
1243             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1244         } else if (impl instanceof ArpProvider) {
1245             arpProvider = (ArpProvider)impl;
1246         } else if (impl instanceof InboundNatProvider) {
1247             inboundNatProvider = (InboundNatProvider)impl;
1248         } else if (impl instanceof OutboundNatProvider) {
1249             outboundNatProvider = (OutboundNatProvider)impl;
1250         } else if (impl instanceof RoutingProvider) {
1251             routingProvider = (RoutingProvider)impl;
1252         } else if (impl instanceof L3ForwardingProvider) {
1253             l3ForwardingProvider = (L3ForwardingProvider)impl;
1254         }
1255     }
1256 }