openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  29 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  30
  31 import com.google.common.base.Preconditions;
  32 import org.osgi.framework.BundleContext;
  33 import org.osgi.framework.ServiceReference;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.net.InetAddress;
  38 import java.net.UnknownHostException;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.ArrayList;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45
  46 /**
  47  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  48  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  49  * as well as the multi-tenant router forwarding provider.
  50  */
  51 public class NeutronL3Adapter implements ConfigInterface {
  52     private static final Logger LOGGER = LoggerFactory.getLogger(NeutronL3Adapter.class);
  53
  54     // The implementation for each of these services is resolved by the OSGi Service Manager
  55     private volatile ConfigurationService configurationService;
  56     private volatile TenantNetworkManager tenantNetworkManager;
  57     private volatile NodeCacheManager nodeCacheManager;
  58     private volatile INeutronNetworkCRUD neutronNetworkCache;
  59     private volatile INeutronSubnetCRUD neutronSubnetCache;
  60     private volatile INeutronPortCRUD neutronPortCache;
  61     private volatile L3ForwardingProvider l3ForwardingProvider;
  62     private volatile InboundNatProvider inboundNatProvider;
  63     private volatile OutboundNatProvider outboundNatProvider;
  64     private volatile ArpProvider arpProvider;
  65     private volatile RoutingProvider routingProvider;
  66
  67     private class FloatIpData {
  68         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  69         private final Long ofPort;        // patch port in br-int used to reach br-ex
  70         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  71         private final String macAddress;  // mac address assigned to neutron port of floating ip
  72         private final String floatingIpAddress;
  73         private final String fixedIpAddress;  // ip address given to tenant vm
  74         private final String neutronRouterMac;
  75
  76         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  77                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  78             this.dpid = dpid;
  79             this.ofPort = ofPort;
  80             this.segId = segId;
  81             this.macAddress = macAddress;
  82             this.floatingIpAddress = floatingIpAddress;
  83             this.fixedIpAddress = fixedIpAddress;
  84             this.neutronRouterMac = neutronRouterMac;
  85         }
  86     }
  87
  88     private Set<String> inboundIpRewriteCache;
  89     private Set<String> outboundIpRewriteCache;
  90     private Set<String> outboundIpRewriteExclusionCache;
  91     private Set<String> routerInterfacesCache;
  92     private Set<String> staticArpEntryCache;
  93     private Set<String> l3ForwardingCache;
  94     private Map<String, String> networkIdToRouterMacCache;
  95     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
  96     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  97     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
  98     private Map<String, FloatIpData> floatIpDataMapCache;
  99     private String externalRouterMac;
 100     private Boolean enabled = false;
 101     private Boolean flgDistributedARPEnabled = true;
 102     private Southbound southbound;
 103
 104     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 105     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 106     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 107     private static final String OWNER_FLOATING_IP = "network:floatingip";
 108     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 109
 110     public NeutronL3Adapter() {
 111         LOGGER.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 112     }
 113
 114     private void initL3AdapterMembers() {
 115         Preconditions.checkNotNull(configurationService);
 116
 117         if (configurationService.isL3ForwardingEnabled()) {
 118             this.inboundIpRewriteCache = new HashSet<>();
 119             this.outboundIpRewriteCache = new HashSet<>();
 120             this.outboundIpRewriteExclusionCache = new HashSet<>();
 121             this.routerInterfacesCache = new HashSet<>();
 122             this.staticArpEntryCache = new HashSet<>();
 123             this.l3ForwardingCache = new HashSet<>();
 124             this.networkIdToRouterMacCache = new HashMap<>();
 125             this.networkIdToRouterIpListCache = new HashMap<>();
 126             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 127             this.neutronPortToDpIdCache = new HashMap<>();
 128             this.floatIpDataMapCache = new HashMap<>();
 129
 130             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 131             if (this.externalRouterMac == null) {
 132                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 133             }
 134
 135             this.enabled = true;
 136             LOGGER.info("OVSDB L3 forwarding is enabled");
 137             if (configurationService.isDistributedArpDisabled()) {
 138                 this.flgDistributedARPEnabled = false;
 139                 LOGGER.info("Distributed ARP responder is disabled");
 140             } else {
 141                 LOGGER.debug("Distributed ARP responder is enabled");
 142             }
 143         } else {
 144             LOGGER.debug("OVSDB L3 forwarding is disabled");
 145         }
 146     }
 147
 148     //
 149     // Callbacks from OVSDB's northbound handlers
 150     //
 151
 152     /**
 153      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 154      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 155      * subscribe so that the mac address associated to the gateway ip address is available.
 156      *
 157      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 158      */
 159     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 160         Preconditions.checkNotNull(externalRouterMacUpdate);
 161
 162         flushExistingIpRewrite();
 163         this.externalRouterMac = externalRouterMacUpdate;
 164         rebuildExistingIpRewrite();
 165     }
 166
 167     /**
 168      * Process the event.
 169      *
 170      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 171      * @param subnet An instance of NeutronSubnet object.
 172      */
 173     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 174         LOGGER.debug("Neutron subnet {} event : {}", action, subnet.toString());
 175         if (!this.enabled) {
 176             return;
 177         }
 178     }
 179
 180     /**
 181      * Process the port event as a router interface event.
 182      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 183      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 184      *
 185      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 186      * @param neutronPort An instance of NeutronPort object.
 187      */
 188     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 189         LOGGER.debug("Neutron port {} event : {}", action, neutronPort.toString());
 190         if (!this.enabled) {
 191             return;
 192         }
 193
 194         final boolean isDelete = action == Action.DELETE;
 195
 196         // Treat the port event as a router interface event if the port belongs to router. This is a
 197         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 198         //
 199         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 200             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 201             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 202                 NeutronRouter_Interface neutronRouterInterface =
 203                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 204                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 205                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 206
 207                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 208             }
 209         } else {
 210             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 211             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 212             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 213             // there.
 214             //
 215             if (!isDelete) {
 216                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 217                     NeutronRouter_Interface neutronRouterInterface =
 218                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 219                     if (neutronRouterInterface != null) {
 220                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 221                     }
 222                 }
 223             }
 224             this.updateL3ForNeutronPort(neutronPort, isDelete);
 225         }
 226     }
 227
 228     /**
 229      * Process the event.
 230      *
 231      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 232      * @param neutronRouter An instance of NeutronRouter object.
 233      */
 234     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 235         LOGGER.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 236         if (!this.enabled) {
 237             return;
 238         }
 239     }
 240
 241     /**
 242      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 243      * delete the ports on the same subnet.
 244      *
 245      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 246      * @param neutronRouter An instance of NeutronRouter object.
 247      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 248      */
 249     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 250                                                   final NeutronRouter_Interface neutronRouterInterface,
 251                                                   Action action) {
 252         LOGGER.debug("Router interface {} got event {}. Subnet {}",
 253                      neutronRouterInterface.getPortUUID(),
 254                      action,
 255                      neutronRouterInterface.getSubnetUUID());
 256         if (!this.enabled) {
 257             return;
 258         }
 259
 260         final boolean isDelete = action == Action.DELETE;
 261
 262         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 263
 264         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 265         // see if they are affected by l3
 266         //
 267         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 268             boolean currPortShouldBeDeleted = false;
 269             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 270             if (isDelete) {
 271                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 272                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 273                         currPortShouldBeDeleted = true;
 274                         break;
 275                     }
 276                 }
 277             }
 278             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 279         }
 280     }
 281
 282     /**
 283      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 284      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 285      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 286      * way around) on OpenFlow Table 100.
 287      *
 288      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 289      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 290      */
 291     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 292                                              Action actionIn) {
 293         Preconditions.checkNotNull(neutronFloatingIP);
 294
 295         LOGGER.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 296                 neutronFloatingIP.getFixedIPAddress(),
 297                 neutronFloatingIP.getFloatingIPAddress(),
 298                 neutronFloatingIP.getFloatingNetworkUUID());
 299         if (!this.enabled) {
 300             return;
 301         }
 302
 303         Action action;
 304
 305         // Consider action to be delete if getFixedIPAddress is null
 306         //
 307         if (neutronFloatingIP.getFixedIPAddress() == null) {
 308             action = Action.DELETE;
 309         } else {
 310             action = actionIn;
 311         }
 312
 313         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 314
 315         if (action != Action.DELETE) {
 316             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 317
 318             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 319             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 320         } else {
 321             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 322             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 323
 324             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 325         }
 326     }
 327
 328     /**
 329      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 330      * ip, otherwise for newer one.
 331      */
 332     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 333         Preconditions.checkNotNull(neutronFloatingIP);
 334
 335         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 336         if (fid == null) {
 337             LOGGER.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 338                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 339             return;
 340         }
 341         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 342                                       action);
 343     }
 344
 345     /**
 346      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 347      * ip, otherwise for newer one.
 348      */
 349     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 350         Preconditions.checkNotNull(neutronFloatingIP);
 351
 352         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 353         if (fid == null) {
 354             LOGGER.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 355                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 356             return;
 357         }
 358         programOutboundIpRewriteStage1(fid, action);
 359     }
 360
 361     private void flushExistingIpRewrite() {
 362         for (FloatIpData fid : floatIpDataMapCache.values()) {
 363             programOutboundIpRewriteStage1(fid, Action.DELETE);
 364         }
 365     }
 366
 367     private void rebuildExistingIpRewrite() {
 368         for (FloatIpData fid : floatIpDataMapCache.values()) {
 369             programOutboundIpRewriteStage1(fid, Action.ADD);
 370         }
 371     }
 372
 373     /**
 374      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 375      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 376      * requests.
 377      */
 378     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 379         Preconditions.checkNotNull(neutronFloatingIP);
 380         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 381         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 382
 383         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 384             LOGGER.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 385                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 386             return;
 387         }
 388
 389         // find bridge Node where floating ip is configured by looking up cache for its port
 390         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 391         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 392         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 393         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 394         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 395         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 396
 397         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 398         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 399                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 400         final String providerSegmentationId = tenantNeutronNetwork != null ?
 401                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 402         final String neutronRouterMac = tenantNeutronNetwork != null ?
 403                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 404
 405         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 406                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 407                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 408                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 409             LOGGER.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 410                     fixedIpAddress,
 411                     floatingIpAddress,
 412                     neutronPortForFloatIp,
 413                     neutronTenantPortUuid,
 414                     providerSegmentationId,
 415                     floatingIpMac,
 416                     neutronRouterMac);
 417             return;
 418         }
 419
 420         // get ofport for patch port in br-int
 421         final Long dpId = nodeIfPair.getLeft();
 422         final Long ofPort = findOFPortForExtPatch(dpId);
 423         if (ofPort == null) {
 424             LOGGER.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 425                     dpId);
 426             return;
 427         }
 428
 429         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 430         //
 431         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 432                 Action.ADD)) {
 433             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 434                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 435             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 436             LOGGER.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 437                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 438                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 439         }
 440     }
 441
 442     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 443         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 444         if (floatIpData == null) {
 445             LOGGER.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 446             return;
 447         }
 448
 449         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 450                 floatIpData.floatingIpAddress, Action.DELETE)) {
 451             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 452             LOGGER.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 453                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 454         }
 455     }
 456
 457     private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 458         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 459             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 460                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 461                 return neutronPort;
 462             }
 463         }
 464         return null;
 465     }
 466
 467     private final Long findOFPortForExtPatch(Long dpId) {
 468         final String brInt = configurationService.getIntegrationBridgeName();
 469         final String brExt = configurationService.getExternalBridgeName();
 470         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 471
 472         Preconditions.checkNotNull(dpId);
 473         Preconditions.checkNotNull(portNameInt);
 474
 475         final long dpidPrimitive = dpId.longValue();
 476         for (Node node : nodeCacheManager.getBridgeNodes()) {
 477             if (dpidPrimitive == southbound.getDataPathId(node)) {
 478                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 479                         southbound.getTerminationPointOfBridge(node, portNameInt);
 480                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 481             }
 482         }
 483         return null;
 484     }
 485
 486     /**
 487      * Process the event.
 488      *
 489      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 490      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 491      */
 492     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 493         LOGGER.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 494         if (!this.enabled) {
 495             return;
 496         }
 497     }
 498
 499     //
 500     // Callbacks from OVSDB's southbound handler
 501     //
 502     /**
 503      * Process the event.
 504      *
 505      * @param bridgeNode An instance of Node object.
 506      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 507      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 508      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 509      * object.
 510      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 511      */
 512     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 513                                      final NeutronNetwork neutronNetwork, Action action) {
 514         LOGGER.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 515                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 516         if (!this.enabled) {
 517             return;
 518         }
 519
 520         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 521         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 522         final Uuid interfaceUuid = intf.getInterfaceUuid();
 523
 524         LOGGER.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 525                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 526
 527         if (neutronPort != null) {
 528             final String neutronPortUuid = neutronPort.getPortUUID();
 529
 530             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 531                     dpId != null && interfaceUuid != null) {
 532                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 533             }
 534
 535             handleNeutronPortEvent(neutronPort, action);
 536         }
 537
 538         if (action == Action.DELETE && interfaceUuid != null) {
 539             handleInterfaceEventDelete(intf, dpId);
 540         }
 541     }
 542
 543     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 544         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 545         LOGGER.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 546                 neutronPortUuid, dpId, interfaceUuid.getValue());
 547     }
 548
 549     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 550         // Remove entry from neutronPortToDpIdCache based on interface uuid
 551         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 552             final String currPortUuid = entry.getKey();
 553             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 554                 LOGGER.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 555                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 556                 neutronPortToDpIdCache.remove(currPortUuid);
 557                 break;
 558             }
 559         }
 560     }
 561
 562     //
 563     // Internal helpers
 564     //
 565     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 566
 567         final String networkUUID = neutronPort.getNetworkUUID();
 568         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 569
 570         // If there is no router interface handling the networkUUID, we are done
 571         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 572             return;
 573         }
 574
 575         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 576         // router interface are handled via handleNeutronRouterInterfaceEvent.
 577         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 578             return;
 579         }
 580
 581         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 582         final String providerSegmentationId = neutronNetwork != null ?
 583                                               neutronNetwork.getProviderSegmentationID() : null;
 584         final String tenantMac = neutronPort.getMacAddress();
 585
 586         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 587             tenantMac == null || tenantMac.isEmpty()) {
 588             // done: go no further w/out all the info needed...
 589             return;
 590         }
 591
 592         final Action action = isDelete ? Action.DELETE : Action.ADD;
 593         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 594         if (nodes.isEmpty()) {
 595             LOGGER.trace("updateL3ForNeutronPort has no nodes to work with");
 596         }
 597         for (Node node : nodes) {
 598             final Long dpid = getDpidForIntegrationBridge(node);
 599             if (dpid == null) {
 600                 continue;
 601             }
 602
 603             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 604                 final String tenantIpStr = neutronIP.getIpAddress();
 605                 if (tenantIpStr.isEmpty()) {
 606                     continue;
 607                 }
 608
 609                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 610                 // still needed when routing to subnets non-local to node (bug 2076).
 611                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 612
 613                 // Configure distributed ARP responder
 614                 if (true == flgDistributedARPEnabled) {
 615                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 616                 }
 617             }
 618         }
 619     }
 620
 621     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 622                                            String macAddress, String ipStr,
 623                                            Action actionForNode) {
 624         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 625         // will look at desired action for node.
 626         //
 627         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 628         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 629
 630         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 631             LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 632                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 633             return;
 634         }
 635         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 636             LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 637                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 638             return;
 639         }
 640
 641         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 642                                                        macAddress, ipStr, actionForNode);
 643         if (status.isSuccess()) {
 644             // Update cache
 645             if (actionForNode == Action.ADD) {
 646                 l3ForwardingCache.add(cacheKey);
 647             } else {
 648                 l3ForwardingCache.remove(cacheKey);
 649             }
 650         }
 651     }
 652
 653     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 654                                              String macAddress,
 655                                              String address,
 656                                              Action actionForNode) {
 657         Status status;
 658         try {
 659             InetAddress inetAddress = InetAddress.getByName(address);
 660             status = l3ForwardingProvider == null ?
 661                      new Status(StatusCode.SUCCESS) :
 662                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 663                                                                       inetAddress, macAddress, actionForNode);
 664         } catch (UnknownHostException e) {
 665             status = new Status(StatusCode.BADREQUEST);
 666         }
 667
 668         if (status.isSuccess()) {
 669             LOGGER.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 670                          l3ForwardingProvider == null ? "skipped" : "programmed",
 671                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 672         } else {
 673             LOGGER.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 674                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 675         }
 676         return status;
 677     }
 678
 679     // --
 680
 681     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 682                                                        Boolean isDelete) {
 683         Preconditions.checkNotNull(destNeutronRouterInterface);
 684
 685         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 686         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 687         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 688         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 689         final NeutronNetwork neutronNetwork = subnet != null ?
 690                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 691         final String destinationSegmentationId = neutronNetwork != null ?
 692                                                  neutronNetwork.getProviderSegmentationID() : null;
 693         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 694         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 695         final String cidr = subnet != null ? subnet.getCidr() : null;
 696         final int mask = getMaskLenFromCidr(cidr);
 697
 698         LOGGER.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 699                      destNeutronRouterInterface, isDelete);
 700
 701         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 702         // the local cache
 703         if (neutronNetwork != null) {
 704             if (macAddress == null || macAddress.isEmpty()) {
 705                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 706             }
 707             if (ipList == null || ipList.isEmpty()) {
 708                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 709             }
 710         }
 711
 712         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 713             cidr == null || cidr.isEmpty() ||
 714             macAddress == null || macAddress.isEmpty() ||
 715             ipList == null || ipList.isEmpty()) {
 716             LOGGER.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 717                          destinationSegmentationId, cidr, macAddress, ipList);
 718             // done: go no further w/out all the info needed...
 719             return;
 720         }
 721
 722         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 723
 724         // Keep cache for finding router's mac from network uuid -- add
 725         //
 726         if (! isDelete) {
 727             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 728             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 729             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 730         }
 731
 732         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 733         if (nodes.isEmpty()) {
 734             LOGGER.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 735         }
 736         for (Node node : nodes) {
 737             final Long dpid = getDpidForIntegrationBridge(node);
 738             if (dpid == null) {
 739                 continue;
 740             }
 741
 742             for (Neutron_IPs neutronIP : ipList) {
 743                 final String ipStr = neutronIP.getIpAddress();
 744                 if (ipStr.isEmpty()) {
 745                     LOGGER.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 746                             node.getNodeId().getValue(), ipStr);
 747                     continue;
 748                 }
 749
 750                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 751                 //
 752                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 753                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 754                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 755                                                               neutronNetwork, destinationSegmentationId,
 756                                                               macAddress, ipStr, mask, actionForNode,
 757                                                               true /*isReflexsive*/);
 758                 }
 759
 760                 if (! isExternal) {
 761                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 762                             actionForNode);
 763                 }
 764                 // Enable ARP responder by default, because router interface needs to be responded always.
 765                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 766             }
 767
 768             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 769             // for the external neutron networks.
 770             //
 771             {
 772                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 773                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 774             }
 775         }
 776
 777         // Keep cache for finding router's mac from network uuid -- remove
 778         //
 779         if (isDelete) {
 780             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 781             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 782             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 783         }
 784     }
 785
 786     private void programFlowForNetworkFromExternal(final Node node,
 787                                                    final Long dpid,
 788                                                    final String destinationSegmentationId,
 789                                                    final String dstMacAddress,
 790                                                    final String destIpStr,
 791                                                    final int destMask,
 792                                                    final Action actionForNode) {
 793         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 794                 dstMacAddress, destIpStr, destMask, actionForNode);
 795     }
 796
 797     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 798                                                            final Long dpid,
 799                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 800                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 801                                                            final NeutronNetwork dstNeutronNetwork,
 802                                                            final String destinationSegmentationId,
 803                                                            final String dstMacAddress,
 804                                                            final String destIpStr,
 805                                                            final int destMask,
 806                                                            final Action actionForNode,
 807                                                            Boolean isReflexsive) {
 808         Preconditions.checkNotNull(srcNeutronRouterInterface);
 809         Preconditions.checkNotNull(dstNeutronRouterInterface);
 810
 811         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 812         if (sourceSubnetId == null) {
 813             LOGGER.error("Could not get provider Subnet ID from router interface {}",
 814                          srcNeutronRouterInterface.getID());
 815             return;
 816         }
 817
 818         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 819         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 820         if (sourceNetworkId == null) {
 821             LOGGER.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 822             return;
 823         }
 824
 825         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 826         if (sourceNetwork == null) {
 827             LOGGER.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 828             return;
 829         }
 830
 831         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 832             // Isolate subnets from different tenants within the same router
 833             return;
 834         }
 835         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 836         if (sourceSegmentationId == null) {
 837             LOGGER.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 838             return;
 839         }
 840         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 841             // Skip 'self'
 842             return;
 843         }
 844
 845         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 846                                      dstMacAddress, destIpStr, destMask, actionForNode);
 847
 848         // Flip roles src->dst; dst->src
 849         if (isReflexsive) {
 850             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 851             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 852             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 853             final String cidr2 = sourceSubnet.getCidr();
 854             final int mask2 = getMaskLenFromCidr(cidr2);
 855
 856             if (cidr2 == null || cidr2.isEmpty() ||
 857                 macAddress2 == null || macAddress2.isEmpty() ||
 858                 ipList2 == null || ipList2.isEmpty()) {
 859                 LOGGER.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 860                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 861                 // done: go no further w/out all the info needed...
 862                 return;
 863             }
 864
 865             for (Neutron_IPs neutronIP2 : ipList2) {
 866                 final String ipStr2 = neutronIP2.getIpAddress();
 867                 if (ipStr2.isEmpty()) {
 868                     continue;
 869                 }
 870                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 871                                                           srcNeutronRouterInterface,
 872                                                           sourceNetwork, sourceSegmentationId,
 873                                                           macAddress2, ipStr2, mask2, actionForNode,
 874                                                           false /*isReflexsive*/);
 875             }
 876         }
 877     }
 878
 879     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 880                                               String destinationSegmentationId,
 881                                               String macAddress, String ipStr, int mask,
 882                                               Action actionForNode) {
 883         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 884         // will look at desired action for node.
 885         //
 886         final String cacheKey = node.getNodeId().getValue() + ":" +
 887                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 888                                 ipStr + "/" + Integer.toString(mask);
 889         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 890
 891         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 892             LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 893                          " action {} is already done",
 894                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 895                          macAddress, ipStr, mask, actionForNode);
 896             return;
 897         }
 898         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 899             LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 900                          " action {} is already done",
 901                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 902                          macAddress, ipStr, mask, actionForNode);
 903             return;
 904         }
 905
 906         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 907                                                           macAddress, ipStr, mask, actionForNode);
 908         if (status.isSuccess()) {
 909             // Update cache
 910             if (actionForNode == Action.ADD) {
 911                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 912                 routerInterfacesCache.add(cacheKey);
 913             } else {
 914                 // TODO: multiTenantAwareRouter.removeInterface(...);
 915                 routerInterfacesCache.remove(cacheKey);
 916             }
 917         }
 918     }
 919
 920     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 921                                                 String destinationSegmentationId,
 922                                                 String macAddress,
 923                                                 String address, int mask,
 924                                                 Action actionForNode) {
 925         Status status;
 926         try {
 927             InetAddress inetAddress = InetAddress.getByName(address);
 928             status = routingProvider == null ?
 929                      new Status(StatusCode.SUCCESS) :
 930                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 931                                                             macAddress, inetAddress, mask, actionForNode);
 932         } catch (UnknownHostException e) {
 933             status = new Status(StatusCode.BADREQUEST);
 934         }
 935
 936         if (status.isSuccess()) {
 937             LOGGER.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 938                          routingProvider == null ? "skipped" : "programmed",
 939                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 940                          actionForNode);
 941         } else {
 942             LOGGER.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 943                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 944                          actionForNode, status);
 945         }
 946         return status;
 947     }
 948
 949     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 950                                            String macAddress, String ipStr,
 951                                            Action action) {
 952         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 953         // will look at desired action for node.
 954         //
 955         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 956         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 957
 958         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 959             LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 960                     dpid, segOrOfPort, macAddress, ipStr, action);
 961             return true;
 962         }
 963         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 964             LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 965                     dpid, segOrOfPort, macAddress, ipStr, action);
 966             return true;
 967         }
 968
 969         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 970         if (status.isSuccess()) {
 971             // Update cache
 972             if (action == Action.ADD) {
 973                 staticArpEntryCache.add(cacheKey);
 974             } else {
 975                 staticArpEntryCache.remove(cacheKey);
 976             }
 977             return true;
 978         }
 979         return false;
 980     }
 981
 982     private Status programStaticArpStage2(Long dpid,
 983                                           String segOrOfPort,
 984                                           String macAddress,
 985                                           String address,
 986                                           Action action) {
 987         Status status;
 988         try {
 989             InetAddress inetAddress = InetAddress.getByName(address);
 990             status = arpProvider == null ?
 991                      new Status(StatusCode.SUCCESS) :
 992                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
 993                                                        macAddress, inetAddress, action);
 994         } catch (UnknownHostException e) {
 995             status = new Status(StatusCode.BADREQUEST);
 996         }
 997
 998         if (status.isSuccess()) {
 999             LOGGER.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1000                          arpProvider == null ? "skipped" : "programmed",
1001                          macAddress, address, dpid, segOrOfPort, action);
1002         } else {
1003             LOGGER.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1004                          macAddress, address, dpid, segOrOfPort, action, status);
1005         }
1006         return status;
1007     }
1008
1009     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1010                                                   String matchAddress, String rewriteAddress,
1011                                                   Action action) {
1012         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1013         // will look at desired action for node.
1014         //
1015         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1016         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1017
1018         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1019             LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1020                     " action {} is already done",
1021                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1022             return true;
1023         }
1024         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1025             LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1026                     " action is already done",
1027                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1028             return true;
1029         }
1030
1031         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1032                 rewriteAddress, action);
1033         if (status.isSuccess()) {
1034             // Update cache
1035             if (action == Action.ADD) {
1036                 inboundIpRewriteCache.add(cacheKey);
1037             } else {
1038                 inboundIpRewriteCache.remove(cacheKey);
1039             }
1040             return true;
1041         }
1042         return false;
1043     }
1044
1045     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1046                                                  String matchAddress, String rewriteAddress,
1047                                                  Action action) {
1048         Status status;
1049         try {
1050             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1051             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1052             status = inboundNatProvider == null ?
1053                     new Status(StatusCode.SUCCESS) :
1054                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1055                             inetMatchAddress, inetRewriteAddress,
1056                             action);
1057         } catch (UnknownHostException e) {
1058             status = new Status(StatusCode.BADREQUEST);
1059         }
1060
1061         if (status.isSuccess()) {
1062             final boolean isSkipped = inboundNatProvider == null;
1063             LOGGER.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1064                     isSkipped ? "skipped" : "programmed",
1065                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1066         } else {
1067             LOGGER.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1068                          " status:{}",
1069                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1070                     status);
1071         }
1072         return status;
1073     }
1074
1075     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1076                                                  Action actionForRewriteExclusion) {
1077         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1078         // will look at desired action for node.
1079         //
1080         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1081         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1082
1083         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1084             LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1085                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1086             return;
1087         }
1088         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1089             LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1090                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1091             return;
1092         }
1093
1094         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1095                                                              actionForRewriteExclusion);
1096         if (status.isSuccess()) {
1097             // Update cache
1098             if (actionForRewriteExclusion == Action.ADD) {
1099                     outboundIpRewriteExclusionCache.add(cacheKey);
1100             } else {
1101                     outboundIpRewriteExclusionCache.remove(cacheKey);
1102             }
1103         }
1104     }
1105
1106     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1107                                                    Action actionForNode) {
1108         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1109                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1110
1111         if (status.isSuccess()) {
1112             final boolean isSkipped = outboundNatProvider == null;
1113             LOGGER.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1114                          isSkipped ? "skipped" : "programmed",
1115                          cidr, node.getNodeId().getValue(), actionForNode);
1116         } else {
1117             LOGGER.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1118                          cidr, node.getNodeId().getValue(), actionForNode, status);
1119         }
1120         return status;
1121     }
1122
1123     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1124         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1125         // will look at desired action for node.
1126         //
1127         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1128         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1129
1130         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1131             LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1132                          "is already done",
1133                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1134             return;
1135         }
1136         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1137             LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1138                          "is already done",
1139                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1140             return;
1141         }
1142
1143         Status status = this.programOutboundIpRewriteStage2(fid, action);
1144         if (status.isSuccess()) {
1145             // Update cache
1146             if (action == Action.ADD) {
1147                 outboundIpRewriteCache.add(cacheKey);
1148             } else {
1149                 outboundIpRewriteCache.remove(cacheKey);
1150             }
1151         }
1152     }
1153
1154     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1155         Status status;
1156         try {
1157             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1158             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1159             status = outboundNatProvider == null ?
1160                     new Status(StatusCode.SUCCESS) :
1161                     outboundNatProvider.programIpRewriteRule(
1162                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1163                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1164         } catch (UnknownHostException e) {
1165             status = new Status(StatusCode.BADREQUEST);
1166         }
1167
1168         if (status.isSuccess()) {
1169             final boolean isSkipped = outboundNatProvider == null;
1170             LOGGER.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1171                             " action {}",
1172                          isSkipped ? "skipped" : "programmed",
1173                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1174         } else {
1175             LOGGER.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1176                          " action {} status:{}",
1177                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1178         }
1179         return status;
1180     }
1181
1182     private int getMaskLenFromCidr(String cidr) {
1183         if (cidr == null) {
1184             return 0;
1185         }
1186         String[] splits = cidr.split("/");
1187         if (splits.length != 2) {
1188             return 0;
1189         }
1190
1191         int result;
1192         try {
1193             result = Integer.parseInt(splits[1].trim());
1194         } catch (NumberFormatException nfe) {
1195             result = 0;
1196         }
1197         return result;
1198     }
1199
1200     private Long getDpidForIntegrationBridge(Node node) {
1201         // Check if node is integration bridge; and only then return its dpid
1202         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1203             return southbound.getDataPathId(node);
1204         }
1205         return null;
1206     }
1207
1208     /**
1209      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1210      *
1211      * @param ofPort the OF port number
1212      * @return the string with encoded OF port (example format "OFPort|999")
1213      */
1214     public static String encodeExcplicitOFPort(Long ofPort) {
1215         return "OFPort|" + ofPort.toString();
1216     }
1217
1218     @Override
1219     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1220         tenantNetworkManager =
1221                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1222         configurationService =
1223                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1224         arpProvider =
1225                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1226         inboundNatProvider =
1227                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1228         outboundNatProvider =
1229                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1230         routingProvider =
1231                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1232         l3ForwardingProvider =
1233                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1234         nodeCacheManager =
1235                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1236         southbound =
1237                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1238
1239         initL3AdapterMembers();
1240     }
1241
1242     @Override
1243     public void setDependencies(Object impl) {
1244         if (impl instanceof INeutronNetworkCRUD) {
1245             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1246         } else if (impl instanceof INeutronPortCRUD) {
1247             neutronPortCache = (INeutronPortCRUD)impl;
1248         } else if (impl instanceof INeutronSubnetCRUD) {
1249             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1250         } else if (impl instanceof ArpProvider) {
1251             arpProvider = (ArpProvider)impl;
1252         } else if (impl instanceof InboundNatProvider) {
1253             inboundNatProvider = (InboundNatProvider)impl;
1254         } else if (impl instanceof OutboundNatProvider) {
1255             outboundNatProvider = (OutboundNatProvider)impl;
1256         } else if (impl instanceof RoutingProvider) {
1257             routingProvider = (RoutingProvider)impl;
1258         } else if (impl instanceof L3ForwardingProvider) {
1259             l3ForwardingProvider = (L3ForwardingProvider)impl;
1260         }
1261     }
1262 }