openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import com.google.common.util.concurrent.FutureCallback;
  34 import com.google.common.util.concurrent.Futures;
  35 import com.google.common.util.concurrent.ListenableFuture;
  36
  37 import org.osgi.framework.BundleContext;
  38 import org.osgi.framework.ServiceReference;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 import java.net.InetAddress;
  43 import java.net.UnknownHostException;
  44 import java.util.HashMap;
  45 import java.util.HashSet;
  46 import java.util.ArrayList;
  47 import java.util.List;
  48 import java.util.Map;
  49 import java.util.Set;
  50 import java.util.concurrent.ExecutorService;
  51 import java.util.concurrent.Executors;
  52
  53 /**
  54  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  55  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  56  * as well as the multi-tenant router forwarding provider.
  57  */
  58 public class NeutronL3Adapter implements ConfigInterface {
  59     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  60
  61     // The implementation for each of these services is resolved by the OSGi Service Manager
  62     private volatile ConfigurationService configurationService;
  63     private volatile TenantNetworkManager tenantNetworkManager;
  64     private volatile NodeCacheManager nodeCacheManager;
  65     private volatile INeutronNetworkCRUD neutronNetworkCache;
  66     private volatile INeutronSubnetCRUD neutronSubnetCache;
  67     private volatile INeutronPortCRUD neutronPortCache;
  68     private volatile L3ForwardingProvider l3ForwardingProvider;
  69     private volatile InboundNatProvider inboundNatProvider;
  70     private volatile OutboundNatProvider outboundNatProvider;
  71     private volatile ArpProvider arpProvider;
  72     private volatile RoutingProvider routingProvider;
  73     private volatile GatewayMacResolver gatewayMacResolver;
  74
  75     private class FloatIpData {
  76         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  77         private final Long ofPort;        // patch port in br-int used to reach br-ex
  78         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  79         private final String macAddress;  // mac address assigned to neutron port of floating ip
  80         private final String floatingIpAddress;
  81         private final String fixedIpAddress;  // ip address given to tenant vm
  82         private final String neutronRouterMac;
  83
  84         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  85                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  86             this.dpid = dpid;
  87             this.ofPort = ofPort;
  88             this.segId = segId;
  89             this.macAddress = macAddress;
  90             this.floatingIpAddress = floatingIpAddress;
  91             this.fixedIpAddress = fixedIpAddress;
  92             this.neutronRouterMac = neutronRouterMac;
  93         }
  94     }
  95
  96     private Set<String> inboundIpRewriteCache;
  97     private Set<String> outboundIpRewriteCache;
  98     private Set<String> outboundIpRewriteExclusionCache;
  99     private Set<String> routerInterfacesCache;
 100     private Set<String> staticArpEntryCache;
 101     private Set<String> l3ForwardingCache;
 102     private Map<String, String> networkIdToRouterMacCache;
 103     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 104     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 105     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 106     private Map<String, FloatIpData> floatIpDataMapCache;
 107     private String externalRouterMac;
 108     private Boolean enabled = false;
 109     private Boolean flgDistributedARPEnabled = true;
 110     private Southbound southbound;
 111     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 112
 113     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 114     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 115     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 116     private static final String OWNER_FLOATING_IP = "network:floatingip";
 117     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 118
 119     public NeutronL3Adapter() {
 120         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 121     }
 122
 123     private void initL3AdapterMembers() {
 124         Preconditions.checkNotNull(configurationService);
 125
 126         if (configurationService.isL3ForwardingEnabled()) {
 127             this.inboundIpRewriteCache = new HashSet<>();
 128             this.outboundIpRewriteCache = new HashSet<>();
 129             this.outboundIpRewriteExclusionCache = new HashSet<>();
 130             this.routerInterfacesCache = new HashSet<>();
 131             this.staticArpEntryCache = new HashSet<>();
 132             this.l3ForwardingCache = new HashSet<>();
 133             this.networkIdToRouterMacCache = new HashMap<>();
 134             this.networkIdToRouterIpListCache = new HashMap<>();
 135             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 136             this.neutronPortToDpIdCache = new HashMap<>();
 137             this.floatIpDataMapCache = new HashMap<>();
 138
 139             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 140             if (this.externalRouterMac == null) {
 141                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 142             }
 143
 144             this.enabled = true;
 145             LOG.info("OVSDB L3 forwarding is enabled");
 146             if (configurationService.isDistributedArpDisabled()) {
 147                 this.flgDistributedARPEnabled = false;
 148                 LOG.info("Distributed ARP responder is disabled");
 149             } else {
 150                 LOG.debug("Distributed ARP responder is enabled");
 151             }
 152         } else {
 153             LOG.debug("OVSDB L3 forwarding is disabled");
 154         }
 155     }
 156
 157     //
 158     // Callbacks from OVSDB's northbound handlers
 159     //
 160
 161     /**
 162      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 163      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 164      * subscribe so that the mac address associated to the gateway ip address is available.
 165      *
 166      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 167      */
 168     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 169         Preconditions.checkNotNull(externalRouterMacUpdate);
 170
 171         flushExistingIpRewrite();
 172         this.externalRouterMac = externalRouterMacUpdate;
 173         rebuildExistingIpRewrite();
 174     }
 175
 176     /**
 177      * Process the event.
 178      *
 179      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 180      * @param subnet An instance of NeutronSubnet object.
 181      */
 182     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 183         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 184     }
 185
 186     /**
 187      * Process the port event as a router interface event.
 188      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 189      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 190      *
 191      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 192      * @param neutronPort An instance of NeutronPort object.
 193      */
 194     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 195         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 196         if (!this.enabled) {
 197             return;
 198         }
 199
 200         final boolean isDelete = action == Action.DELETE;
 201
 202         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 203             if(!isDelete){
 204                 Node externalBridgeNode = getExternalBridgeNode();
 205                 if(externalBridgeNode != null){
 206                     LOG.info("Port {} is network router gateway interface, "
 207                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 208                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 209                 }else{
 210                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 211                 }
 212             }else{
 213                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 214
 215                 if(externalNetwork != null){
 216                     if(externalNetwork.isRouterExternal()){
 217                         final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 218                         // TODO support IPv6
 219                         if (externalSubnet != null &&
 220                             externalSubnet.getIpVersion() == 4) {
 221                             gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 222                         }
 223                     }
 224                 }
 225             }
 226         }
 227
 228         // Treat the port event as a router interface event if the port belongs to router. This is a
 229         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 230         //
 231         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 232             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 233
 234             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 235                 NeutronRouter_Interface neutronRouterInterface =
 236                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 237                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 238                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 239
 240                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 241             }
 242         } else {
 243             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 244             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 245             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 246             // there.
 247             //
 248             if (!isDelete) {
 249                 if (neutronPort.getFixedIPs() != null) {
 250                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 251                         NeutronRouter_Interface neutronRouterInterface =
 252                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 253                         if (neutronRouterInterface != null) {
 254                             this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 255                         }
 256                     }
 257                 }
 258             }
 259             this.updateL3ForNeutronPort(neutronPort, isDelete);
 260         }
 261     }
 262
 263     /**
 264      * Process the event.
 265      *
 266      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 267      * @param neutronRouter An instance of NeutronRouter object.
 268      */
 269     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 270         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 271     }
 272
 273     /**
 274      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 275      * delete the ports on the same subnet.
 276      *
 277      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 278      * @param neutronRouter An instance of NeutronRouter object.
 279      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 280      */
 281     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 282                                                   final NeutronRouter_Interface neutronRouterInterface,
 283                                                   Action action) {
 284         LOG.debug("Router interface {} got event {}. Subnet {}",
 285                      neutronRouterInterface.getPortUUID(),
 286                      action,
 287                      neutronRouterInterface.getSubnetUUID());
 288         if (!this.enabled) {
 289             return;
 290         }
 291
 292         final boolean isDelete = action == Action.DELETE;
 293
 294         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 295
 296         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 297         // see if they are affected by l3
 298         //
 299         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 300             boolean currPortShouldBeDeleted = false;
 301             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 302             if (isDelete) {
 303                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 304                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 305                         currPortShouldBeDeleted = true;
 306                         break;
 307                     }
 308                 }
 309             }
 310             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 311         }
 312     }
 313
 314     /**
 315      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 316      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 317      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 318      * way around) on OpenFlow Table 100.
 319      *
 320      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 321      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 322      */
 323     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 324                                              Action actionIn) {
 325         Preconditions.checkNotNull(neutronFloatingIP);
 326
 327         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 328                 neutronFloatingIP.getFixedIPAddress(),
 329                 neutronFloatingIP.getFloatingIPAddress(),
 330                 neutronFloatingIP.getFloatingNetworkUUID());
 331         if (!this.enabled) {
 332             return;
 333         }
 334
 335         Action action;
 336
 337         // Consider action to be delete if getFixedIPAddress is null
 338         //
 339         if (neutronFloatingIP.getFixedIPAddress() == null) {
 340             action = Action.DELETE;
 341         } else {
 342             action = actionIn;
 343         }
 344
 345         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 346
 347         if (action != Action.DELETE) {
 348             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 349
 350             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 351             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 352         } else {
 353             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 354             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 355
 356             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 357         }
 358     }
 359
 360     /**
 361      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 362      * ip, otherwise for newer one.
 363      */
 364     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 365         Preconditions.checkNotNull(neutronFloatingIP);
 366
 367         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 368         if (fid == null) {
 369             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 370                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 371             return;
 372         }
 373         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 374                                       action);
 375     }
 376
 377     /**
 378      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 379      * ip, otherwise for newer one.
 380      */
 381     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 382         Preconditions.checkNotNull(neutronFloatingIP);
 383
 384         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 385         if (fid == null) {
 386             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 387                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 388             return;
 389         }
 390         programOutboundIpRewriteStage1(fid, action);
 391     }
 392
 393     private void flushExistingIpRewrite() {
 394         for (FloatIpData fid : floatIpDataMapCache.values()) {
 395             programOutboundIpRewriteStage1(fid, Action.DELETE);
 396         }
 397     }
 398
 399     private void rebuildExistingIpRewrite() {
 400         for (FloatIpData fid : floatIpDataMapCache.values()) {
 401             programOutboundIpRewriteStage1(fid, Action.ADD);
 402         }
 403     }
 404
 405     /**
 406      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 407      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 408      * requests.
 409      */
 410     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 411         Preconditions.checkNotNull(neutronFloatingIP);
 412         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 413         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 414
 415         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 416             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 417                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 418             return;
 419         }
 420
 421         // find bridge Node where floating ip is configured by looking up cache for its port
 422         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 423         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 424         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 425         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 426         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 427         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 428
 429         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 430         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 431                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 432         final String providerSegmentationId = tenantNeutronNetwork != null ?
 433                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 434         final String neutronRouterMac = tenantNeutronNetwork != null ?
 435                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 436
 437         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 438                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 439                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 440                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 441             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 442                     fixedIpAddress,
 443                     floatingIpAddress,
 444                     neutronPortForFloatIp,
 445                     neutronTenantPortUuid,
 446                     providerSegmentationId,
 447                     floatingIpMac,
 448                     neutronRouterMac);
 449             return;
 450         }
 451
 452         // get ofport for patch port in br-int
 453         final Long dpId = nodeIfPair.getLeft();
 454         final Long ofPort = findOFPortForExtPatch(dpId);
 455         if (ofPort == null) {
 456             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 457                     dpId);
 458             return;
 459         }
 460
 461         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 462         //
 463         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 464                 Action.ADD)) {
 465             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 466                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 467             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 468             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 469                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 470                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 471         }
 472     }
 473
 474     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 475         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 476         if (floatIpData == null) {
 477             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 478             return;
 479         }
 480
 481         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 482                 floatIpData.floatingIpAddress, Action.DELETE)) {
 483             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 484             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 485                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 486         }
 487     }
 488
 489     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 490         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 491             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 492                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 493                 return neutronPort;
 494             }
 495         }
 496         return null;
 497     }
 498
 499     private Long findOFPortForExtPatch(Long dpId) {
 500         final String brInt = configurationService.getIntegrationBridgeName();
 501         final String brExt = configurationService.getExternalBridgeName();
 502         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 503
 504         Preconditions.checkNotNull(dpId);
 505         Preconditions.checkNotNull(portNameInt);
 506
 507         final long dpidPrimitive = dpId;
 508         for (Node node : nodeCacheManager.getBridgeNodes()) {
 509             if (dpidPrimitive == southbound.getDataPathId(node)) {
 510                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 511                         southbound.getTerminationPointOfBridge(node, portNameInt);
 512                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 513             }
 514         }
 515         return null;
 516     }
 517
 518     /**
 519      * Process the event.
 520      *
 521      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 522      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 523      */
 524     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 525         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 526     }
 527
 528     //
 529     // Callbacks from OVSDB's southbound handler
 530     //
 531     /**
 532      * Process the event.
 533      *
 534      * @param bridgeNode An instance of Node object.
 535      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 536      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 537      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 538      * object.
 539      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 540      */
 541     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 542                                      final NeutronNetwork neutronNetwork, Action action) {
 543         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 544                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 545         if (!this.enabled) {
 546             return;
 547         }
 548
 549         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 550         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 551         final Uuid interfaceUuid = intf.getInterfaceUuid();
 552
 553         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 554                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 555
 556         if (neutronPort != null) {
 557             final String neutronPortUuid = neutronPort.getPortUUID();
 558
 559             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 560                     dpId != null && interfaceUuid != null) {
 561                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 562             }
 563
 564             handleNeutronPortEvent(neutronPort, action);
 565         }
 566
 567         if (action == Action.DELETE && interfaceUuid != null) {
 568             handleInterfaceEventDelete(intf, dpId);
 569         }
 570     }
 571
 572     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 573         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 574         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 575                 neutronPortUuid, dpId, interfaceUuid.getValue());
 576     }
 577
 578     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 579         // Remove entry from neutronPortToDpIdCache based on interface uuid
 580         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 581             final String currPortUuid = entry.getKey();
 582             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 583                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 584                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 585                 neutronPortToDpIdCache.remove(currPortUuid);
 586                 break;
 587             }
 588         }
 589     }
 590
 591     //
 592     // Internal helpers
 593     //
 594     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 595
 596         final String networkUUID = neutronPort.getNetworkUUID();
 597         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 598
 599         // If there is no router interface handling the networkUUID, we are done
 600         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 601             return;
 602         }
 603
 604         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 605         // router interface are handled via handleNeutronRouterInterfaceEvent.
 606         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 607             return;
 608         }
 609
 610         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 611         final String providerSegmentationId = neutronNetwork != null ?
 612                                               neutronNetwork.getProviderSegmentationID() : null;
 613         final String tenantMac = neutronPort.getMacAddress();
 614
 615         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 616             tenantMac == null || tenantMac.isEmpty()) {
 617             // done: go no further w/out all the info needed...
 618             return;
 619         }
 620
 621         final Action action = isDelete ? Action.DELETE : Action.ADD;
 622         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 623         if (nodes.isEmpty()) {
 624             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 625         }
 626         for (Node node : nodes) {
 627             final Long dpid = getDpidForIntegrationBridge(node);
 628             if (dpid == null) {
 629                 continue;
 630             }
 631
 632             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 633                 final String tenantIpStr = neutronIP.getIpAddress();
 634                 if (tenantIpStr.isEmpty()) {
 635                     continue;
 636                 }
 637
 638                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 639                 // still needed when routing to subnets non-local to node (bug 2076).
 640                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 641
 642                 // Configure distributed ARP responder
 643                 if (flgDistributedARPEnabled) {
 644                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 645                 }
 646             }
 647         }
 648     }
 649
 650     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 651                                            String macAddress, String ipStr,
 652                                            Action actionForNode) {
 653         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 654         // will look at desired action for node.
 655
 656         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 657         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 658
 659         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 660             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 661                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 662             return;
 663         }
 664         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 665             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 666                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 667             return;
 668         }
 669
 670         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 671                                                        macAddress, ipStr, actionForNode);
 672         if (status.isSuccess()) {
 673             // Update cache
 674             if (actionForNode == Action.ADD) {
 675                 l3ForwardingCache.add(cacheKey);
 676             } else {
 677                 l3ForwardingCache.remove(cacheKey);
 678             }
 679         }
 680     }
 681
 682     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 683                                              String macAddress,
 684                                              String address,
 685                                              Action actionForNode) {
 686         Status status;
 687         try {
 688             InetAddress inetAddress = InetAddress.getByName(address);
 689             status = l3ForwardingProvider == null ?
 690                      new Status(StatusCode.SUCCESS) :
 691                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 692                                                                       inetAddress, macAddress, actionForNode);
 693         } catch (UnknownHostException e) {
 694             status = new Status(StatusCode.BADREQUEST);
 695         }
 696
 697         if (status.isSuccess()) {
 698             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 699                          l3ForwardingProvider == null ? "skipped" : "programmed",
 700                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 701         } else {
 702             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 703                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 704         }
 705         return status;
 706     }
 707
 708     // --
 709
 710     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 711                                                        Boolean isDelete) {
 712         Preconditions.checkNotNull(destNeutronRouterInterface);
 713
 714         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 715         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 716         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 717         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 718         final NeutronNetwork neutronNetwork = subnet != null ?
 719                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 720         final String destinationSegmentationId = neutronNetwork != null ?
 721                                                  neutronNetwork.getProviderSegmentationID() : null;
 722         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 723         final String cidr = subnet != null ? subnet.getCidr() : null;
 724         final int mask = getMaskLenFromCidr(cidr);
 725
 726         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 727                      destNeutronRouterInterface, isDelete);
 728
 729         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 730         // the local cache
 731         if (neutronNetwork != null) {
 732             if (macAddress == null || macAddress.isEmpty()) {
 733                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 734             }
 735             if (ipList == null || ipList.isEmpty()) {
 736                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 737             }
 738         }
 739
 740         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 741             cidr == null || cidr.isEmpty() ||
 742             macAddress == null || macAddress.isEmpty() ||
 743             ipList == null || ipList.isEmpty()) {
 744             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 745                          destinationSegmentationId, cidr, macAddress, ipList);
 746             // done: go no further w/out all the info needed...
 747             return;
 748         }
 749
 750         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 751
 752         // Keep cache for finding router's mac from network uuid -- add
 753         //
 754         if (! isDelete) {
 755             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 756             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 757             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 758         }
 759
 760         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 761         if (nodes.isEmpty()) {
 762             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 763         }
 764         for (Node node : nodes) {
 765             final Long dpid = getDpidForIntegrationBridge(node);
 766             if (dpid == null) {
 767                 continue;
 768             }
 769
 770             for (Neutron_IPs neutronIP : ipList) {
 771                 final String ipStr = neutronIP.getIpAddress();
 772                 if (ipStr.isEmpty()) {
 773                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 774                             node.getNodeId().getValue(), ipStr);
 775                     continue;
 776                 }
 777
 778                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 779                 //
 780                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 781                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 782                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 783                                                               neutronNetwork, destinationSegmentationId,
 784                                                               macAddress, ipStr, mask, actionForNode,
 785                                                               true /*isReflexsive*/);
 786                 }
 787
 788                 if (! isExternal) {
 789                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 790                             actionForNode);
 791                 }
 792                 // Enable ARP responder by default, because router interface needs to be responded always.
 793                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 794             }
 795
 796             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 797             // for the external neutron networks.
 798             //
 799             {
 800                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 801                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 802             }
 803         }
 804
 805         // Keep cache for finding router's mac from network uuid -- remove
 806         //
 807         if (isDelete) {
 808             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 809             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 810             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 811         }
 812     }
 813
 814     private void programFlowForNetworkFromExternal(final Node node,
 815                                                    final Long dpid,
 816                                                    final String destinationSegmentationId,
 817                                                    final String dstMacAddress,
 818                                                    final String destIpStr,
 819                                                    final int destMask,
 820                                                    final Action actionForNode) {
 821         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 822                 dstMacAddress, destIpStr, destMask, actionForNode);
 823     }
 824
 825     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 826                                                            final Long dpid,
 827                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 828                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 829                                                            final NeutronNetwork dstNeutronNetwork,
 830                                                            final String destinationSegmentationId,
 831                                                            final String dstMacAddress,
 832                                                            final String destIpStr,
 833                                                            final int destMask,
 834                                                            final Action actionForNode,
 835                                                            Boolean isReflexsive) {
 836         Preconditions.checkNotNull(srcNeutronRouterInterface);
 837         Preconditions.checkNotNull(dstNeutronRouterInterface);
 838
 839         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 840         if (sourceSubnetId == null) {
 841             LOG.error("Could not get provider Subnet ID from router interface {}",
 842                          srcNeutronRouterInterface.getID());
 843             return;
 844         }
 845
 846         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 847         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 848         if (sourceNetworkId == null) {
 849             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 850             return;
 851         }
 852
 853         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 854         if (sourceNetwork == null) {
 855             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 856             return;
 857         }
 858
 859         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 860             // Isolate subnets from different tenants within the same router
 861             return;
 862         }
 863         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 864         if (sourceSegmentationId == null) {
 865             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 866             return;
 867         }
 868         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 869             // Skip 'self'
 870             return;
 871         }
 872
 873         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 874                                      dstMacAddress, destIpStr, destMask, actionForNode);
 875
 876         // Flip roles src->dst; dst->src
 877         if (isReflexsive) {
 878             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 879             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 880             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 881             final String cidr2 = sourceSubnet.getCidr();
 882             final int mask2 = getMaskLenFromCidr(cidr2);
 883
 884             if (cidr2 == null || cidr2.isEmpty() ||
 885                 macAddress2 == null || macAddress2.isEmpty() ||
 886                 ipList2 == null || ipList2.isEmpty()) {
 887                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 888                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 889                 // done: go no further w/out all the info needed...
 890                 return;
 891             }
 892
 893             for (Neutron_IPs neutronIP2 : ipList2) {
 894                 final String ipStr2 = neutronIP2.getIpAddress();
 895                 if (ipStr2.isEmpty()) {
 896                     continue;
 897                 }
 898                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 899                                                           srcNeutronRouterInterface,
 900                                                           sourceNetwork, sourceSegmentationId,
 901                                                           macAddress2, ipStr2, mask2, actionForNode,
 902                                                           false /*isReflexsive*/);
 903             }
 904         }
 905     }
 906
 907     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 908                                               String destinationSegmentationId,
 909                                               String macAddress, String ipStr, int mask,
 910                                               Action actionForNode) {
 911         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 912         // will look at desired action for node.
 913         //
 914         final String cacheKey = node.getNodeId().getValue() + ":" +
 915                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 916                                 ipStr + "/" + Integer.toString(mask);
 917         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 918
 919         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 920             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 921                          " action {} is already done",
 922                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 923                          macAddress, ipStr, mask, actionForNode);
 924             return;
 925         }
 926         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 927             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 928                          " action {} is already done",
 929                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 930                          macAddress, ipStr, mask, actionForNode);
 931             return;
 932         }
 933
 934         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 935                                                           macAddress, ipStr, mask, actionForNode);
 936         if (status.isSuccess()) {
 937             // Update cache
 938             if (actionForNode == Action.ADD) {
 939                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 940                 routerInterfacesCache.add(cacheKey);
 941             } else {
 942                 // TODO: multiTenantAwareRouter.removeInterface(...);
 943                 routerInterfacesCache.remove(cacheKey);
 944             }
 945         }
 946     }
 947
 948     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 949                                                 String destinationSegmentationId,
 950                                                 String macAddress,
 951                                                 String address, int mask,
 952                                                 Action actionForNode) {
 953         Status status;
 954         try {
 955             InetAddress inetAddress = InetAddress.getByName(address);
 956             status = routingProvider == null ?
 957                      new Status(StatusCode.SUCCESS) :
 958                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 959                                                             macAddress, inetAddress, mask, actionForNode);
 960         } catch (UnknownHostException e) {
 961             status = new Status(StatusCode.BADREQUEST);
 962         }
 963
 964         if (status.isSuccess()) {
 965             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 966                          routingProvider == null ? "skipped" : "programmed",
 967                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 968                          actionForNode);
 969         } else {
 970             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 971                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 972                          actionForNode, status);
 973         }
 974         return status;
 975     }
 976
 977     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 978                                            String macAddress, String ipStr,
 979                                            Action action) {
 980         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 981         // will look at desired action for node.
 982         //
 983         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 984         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 985
 986         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 987             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 988                     dpid, segOrOfPort, macAddress, ipStr, action);
 989             return true;
 990         }
 991         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 992             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 993                     dpid, segOrOfPort, macAddress, ipStr, action);
 994             return true;
 995         }
 996
 997         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 998         if (status.isSuccess()) {
 999             // Update cache
1000             if (action == Action.ADD) {
1001                 staticArpEntryCache.add(cacheKey);
1002             } else {
1003                 staticArpEntryCache.remove(cacheKey);
1004             }
1005             return true;
1006         }
1007         return false;
1008     }
1009
1010     private Status programStaticArpStage2(Long dpid,
1011                                           String segOrOfPort,
1012                                           String macAddress,
1013                                           String address,
1014                                           Action action) {
1015         Status status;
1016         try {
1017             InetAddress inetAddress = InetAddress.getByName(address);
1018             status = arpProvider == null ?
1019                      new Status(StatusCode.SUCCESS) :
1020                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1021                                                        macAddress, inetAddress, action);
1022         } catch (UnknownHostException e) {
1023             status = new Status(StatusCode.BADREQUEST);
1024         }
1025
1026         if (status.isSuccess()) {
1027             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1028                          arpProvider == null ? "skipped" : "programmed",
1029                          macAddress, address, dpid, segOrOfPort, action);
1030         } else {
1031             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1032                          macAddress, address, dpid, segOrOfPort, action, status);
1033         }
1034         return status;
1035     }
1036
1037     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1038                                                   String matchAddress, String rewriteAddress,
1039                                                   Action action) {
1040         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1041         // will look at desired action for node.
1042         //
1043         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1044         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1045
1046         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1047             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1048                     " action {} is already done",
1049                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1050             return true;
1051         }
1052         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1053             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1054                     " action is already done",
1055                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1056             return true;
1057         }
1058
1059         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1060                 rewriteAddress, action);
1061         if (status.isSuccess()) {
1062             // Update cache
1063             if (action == Action.ADD) {
1064                 inboundIpRewriteCache.add(cacheKey);
1065             } else {
1066                 inboundIpRewriteCache.remove(cacheKey);
1067             }
1068             return true;
1069         }
1070         return false;
1071     }
1072
1073     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1074                                                  String matchAddress, String rewriteAddress,
1075                                                  Action action) {
1076         Status status;
1077         try {
1078             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1079             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1080             status = inboundNatProvider == null ?
1081                     new Status(StatusCode.SUCCESS) :
1082                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1083                             inetMatchAddress, inetRewriteAddress,
1084                             action);
1085         } catch (UnknownHostException e) {
1086             status = new Status(StatusCode.BADREQUEST);
1087         }
1088
1089         if (status.isSuccess()) {
1090             final boolean isSkipped = inboundNatProvider == null;
1091             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1092                     isSkipped ? "skipped" : "programmed",
1093                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1094         } else {
1095             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1096                          " status:{}",
1097                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1098                     status);
1099         }
1100         return status;
1101     }
1102
1103     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1104                                                  Action actionForRewriteExclusion) {
1105         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1106         // will look at desired action for node.
1107         //
1108         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1109         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1110
1111         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1112             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1113                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1114             return;
1115         }
1116         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1117             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1118                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1119             return;
1120         }
1121
1122         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1123                                                              actionForRewriteExclusion);
1124         if (status.isSuccess()) {
1125             // Update cache
1126             if (actionForRewriteExclusion == Action.ADD) {
1127                     outboundIpRewriteExclusionCache.add(cacheKey);
1128             } else {
1129                     outboundIpRewriteExclusionCache.remove(cacheKey);
1130             }
1131         }
1132     }
1133
1134     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1135                                                    Action actionForNode) {
1136         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1137                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1138
1139         if (status.isSuccess()) {
1140             final boolean isSkipped = outboundNatProvider == null;
1141             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1142                          isSkipped ? "skipped" : "programmed",
1143                          cidr, node.getNodeId().getValue(), actionForNode);
1144         } else {
1145             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1146                          cidr, node.getNodeId().getValue(), actionForNode, status);
1147         }
1148         return status;
1149     }
1150
1151     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1152         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1153         // will look at desired action for node.
1154         //
1155         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1156         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1157
1158         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1159             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1160                          "is already done",
1161                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1162             return;
1163         }
1164         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1165             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1166                          "is already done",
1167                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1168             return;
1169         }
1170
1171         Status status = this.programOutboundIpRewriteStage2(fid, action);
1172         if (status.isSuccess()) {
1173             // Update cache
1174             if (action == Action.ADD) {
1175                 outboundIpRewriteCache.add(cacheKey);
1176             } else {
1177                 outboundIpRewriteCache.remove(cacheKey);
1178             }
1179         }
1180     }
1181
1182     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1183         Status status;
1184         try {
1185             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1186             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1187             status = outboundNatProvider == null ?
1188                     new Status(StatusCode.SUCCESS) :
1189                     outboundNatProvider.programIpRewriteRule(
1190                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1191                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1192         } catch (UnknownHostException e) {
1193             status = new Status(StatusCode.BADREQUEST);
1194         }
1195
1196         if (status.isSuccess()) {
1197             final boolean isSkipped = outboundNatProvider == null;
1198             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1199                             " action {}",
1200                          isSkipped ? "skipped" : "programmed",
1201                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1202         } else {
1203             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1204                          " action {} status:{}",
1205                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1206         }
1207         return status;
1208     }
1209
1210     private int getMaskLenFromCidr(String cidr) {
1211         if (cidr == null) {
1212             return 0;
1213         }
1214         String[] splits = cidr.split("/");
1215         if (splits.length != 2) {
1216             return 0;
1217         }
1218
1219         int result;
1220         try {
1221             result = Integer.parseInt(splits[1].trim());
1222         } catch (NumberFormatException nfe) {
1223             result = 0;
1224         }
1225         return result;
1226     }
1227
1228     private Long getDpidForIntegrationBridge(Node node) {
1229         // Check if node is integration bridge; and only then return its dpid
1230         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1231             return southbound.getDataPathId(node);
1232         }
1233         return null;
1234     }
1235
1236     private Long getDpidForExternalBridge(Node node) {
1237         // Check if node is integration bridge; and only then return its dpid
1238         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1239             return southbound.getDataPathId(node);
1240         }
1241         return null;
1242     }
1243
1244     private Node getExternalBridgeNode(){
1245         //Pickup the first node that has external bridge (br-ex).
1246         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1247         //the external network is reachable from every br-ex
1248         // TODO: Consider other deployment scenario, and thing of better solution.
1249         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1250         for(Node node : allBridges){
1251             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1252                 return node;
1253             }
1254         }
1255         return null;
1256     }
1257
1258     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1259         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1260             String subnetUUID = neutronIPs.getSubnetUUID();
1261             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1262             if (extSubnet.getGatewayIP() == null) {
1263                 continue;
1264             }
1265             return extSubnet;
1266         }
1267         return null;
1268     }
1269
1270     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1271
1272         Preconditions.checkNotNull(node);
1273         Preconditions.checkNotNull(gatewayPort);
1274         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1275
1276         if(externalNetwork != null){
1277             if(externalNetwork.isRouterExternal()){
1278                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1279
1280                 // TODO: address IPv6 case.
1281                 if (externalSubnet != null &&
1282                     externalSubnet.getIpVersion() == 4) {
1283                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1284                     ListenableFuture<MacAddress> gatewayMacAddress =
1285                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1286                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1287                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1288                                                              new MacAddress(gatewayPort.getMacAddress()),
1289                                                              true);
1290                     if(gatewayMacAddress != null){
1291                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1292                             @Override
1293                             public void onSuccess(MacAddress result) {
1294                                 if(result != null){
1295                                     if(!result.getValue().equals(externalRouterMac)){
1296                                         updateExternalRouterMac(result.getValue());
1297                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1298                                     }
1299                                 }else{
1300                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1301                                 }
1302                             }
1303
1304                             @Override
1305                             public void onFailure(Throwable t) {
1306                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1307                             }
1308                         }, gatewayMacResolverPool);
1309                     }
1310                 } else {
1311                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1312                 }
1313             }
1314         }else{
1315             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1316         }
1317     }
1318
1319     /**
1320      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1321      *
1322      * @param ofPort the OF port number
1323      * @return the string with encoded OF port (example format "OFPort|999")
1324      */
1325     public static String encodeExcplicitOFPort(Long ofPort) {
1326         return "OFPort|" + ofPort.toString();
1327     }
1328
1329     @Override
1330     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1331         tenantNetworkManager =
1332                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1333         configurationService =
1334                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1335         arpProvider =
1336                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1337         inboundNatProvider =
1338                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1339         outboundNatProvider =
1340                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1341         routingProvider =
1342                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1343         l3ForwardingProvider =
1344                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1345         nodeCacheManager =
1346                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1347         southbound =
1348                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1349         gatewayMacResolver =
1350                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1351         initL3AdapterMembers();
1352     }
1353
1354     @Override
1355     public void setDependencies(Object impl) {
1356         if (impl instanceof INeutronNetworkCRUD) {
1357             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1358         } else if (impl instanceof INeutronPortCRUD) {
1359             neutronPortCache = (INeutronPortCRUD)impl;
1360         } else if (impl instanceof INeutronSubnetCRUD) {
1361             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1362         } else if (impl instanceof ArpProvider) {
1363             arpProvider = (ArpProvider)impl;
1364         } else if (impl instanceof InboundNatProvider) {
1365             inboundNatProvider = (InboundNatProvider)impl;
1366         } else if (impl instanceof OutboundNatProvider) {
1367             outboundNatProvider = (OutboundNatProvider)impl;
1368         } else if (impl instanceof RoutingProvider) {
1369             routingProvider = (RoutingProvider)impl;
1370         } else if (impl instanceof L3ForwardingProvider) {
1371             l3ForwardingProvider = (L3ForwardingProvider)impl;
1372         }else if (impl instanceof GatewayMacResolver) {
1373             gatewayMacResolver = (GatewayMacResolver)impl;
1374         }
1375     }
1376 }