openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  31 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  32
  33 import com.google.common.base.Preconditions;
  34 import com.google.common.util.concurrent.FutureCallback;
  35 import com.google.common.util.concurrent.Futures;
  36 import com.google.common.util.concurrent.ListenableFuture;
  37
  38 import org.osgi.framework.BundleContext;
  39 import org.osgi.framework.ServiceReference;
  40 import org.slf4j.Logger;
  41 import org.slf4j.LoggerFactory;
  42
  43 import java.net.InetAddress;
  44 import java.net.UnknownHostException;
  45 import java.util.HashMap;
  46 import java.util.HashSet;
  47 import java.util.ArrayList;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.Set;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53
  54 /**
  55  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  56  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  57  * as well as the multi-tenant router forwarding provider.
  58  */
  59 public class NeutronL3Adapter implements ConfigInterface {
  60     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  61
  62     // The implementation for each of these services is resolved by the OSGi Service Manager
  63     private volatile ConfigurationService configurationService;
  64     private volatile TenantNetworkManager tenantNetworkManager;
  65     private volatile NodeCacheManager nodeCacheManager;
  66     private volatile INeutronNetworkCRUD neutronNetworkCache;
  67     private volatile INeutronSubnetCRUD neutronSubnetCache;
  68     private volatile INeutronPortCRUD neutronPortCache;
  69     private volatile L3ForwardingProvider l3ForwardingProvider;
  70     private volatile InboundNatProvider inboundNatProvider;
  71     private volatile OutboundNatProvider outboundNatProvider;
  72     private volatile ArpProvider arpProvider;
  73     private volatile RoutingProvider routingProvider;
  74     private volatile GatewayMacResolver gatewayMacResolver;
  75
  76     private class FloatIpData {
  77         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  78         private final Long ofPort;        // patch port in br-int used to reach br-ex
  79         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  80         private final String macAddress;  // mac address assigned to neutron port of floating ip
  81         private final String floatingIpAddress;
  82         private final String fixedIpAddress;  // ip address given to tenant vm
  83         private final String neutronRouterMac;
  84
  85         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  86                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  87             this.dpid = dpid;
  88             this.ofPort = ofPort;
  89             this.segId = segId;
  90             this.macAddress = macAddress;
  91             this.floatingIpAddress = floatingIpAddress;
  92             this.fixedIpAddress = fixedIpAddress;
  93             this.neutronRouterMac = neutronRouterMac;
  94         }
  95     }
  96
  97     private Set<String> inboundIpRewriteCache;
  98     private Set<String> outboundIpRewriteCache;
  99     private Set<String> outboundIpRewriteExclusionCache;
 100     private Set<String> routerInterfacesCache;
 101     private Set<String> staticArpEntryCache;
 102     private Set<String> l3ForwardingCache;
 103     private Map<String, String> networkIdToRouterMacCache;
 104     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 105     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 106     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 107     private Map<String, FloatIpData> floatIpDataMapCache;
 108     private String externalRouterMac;
 109     private Boolean enabled = false;
 110     private Boolean flgDistributedARPEnabled = true;
 111     private Southbound southbound;
 112     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 113
 114     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 115     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 116     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 117     private static final String OWNER_FLOATING_IP = "network:floatingip";
 118     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 119
 120     public NeutronL3Adapter() {
 121         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 122     }
 123
 124     private void initL3AdapterMembers() {
 125         Preconditions.checkNotNull(configurationService);
 126
 127         if (configurationService.isL3ForwardingEnabled()) {
 128             this.inboundIpRewriteCache = new HashSet<>();
 129             this.outboundIpRewriteCache = new HashSet<>();
 130             this.outboundIpRewriteExclusionCache = new HashSet<>();
 131             this.routerInterfacesCache = new HashSet<>();
 132             this.staticArpEntryCache = new HashSet<>();
 133             this.l3ForwardingCache = new HashSet<>();
 134             this.networkIdToRouterMacCache = new HashMap<>();
 135             this.networkIdToRouterIpListCache = new HashMap<>();
 136             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 137             this.neutronPortToDpIdCache = new HashMap<>();
 138             this.floatIpDataMapCache = new HashMap<>();
 139
 140             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 141             if (this.externalRouterMac == null) {
 142                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 143             }
 144
 145             this.enabled = true;
 146             LOG.info("OVSDB L3 forwarding is enabled");
 147             if (configurationService.isDistributedArpDisabled()) {
 148                 this.flgDistributedARPEnabled = false;
 149                 LOG.info("Distributed ARP responder is disabled");
 150             } else {
 151                 LOG.debug("Distributed ARP responder is enabled");
 152             }
 153         } else {
 154             LOG.debug("OVSDB L3 forwarding is disabled");
 155         }
 156     }
 157
 158     //
 159     // Callbacks from OVSDB's northbound handlers
 160     //
 161
 162     /**
 163      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 164      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 165      * subscribe so that the mac address associated to the gateway ip address is available.
 166      *
 167      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 168      */
 169     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 170         Preconditions.checkNotNull(externalRouterMacUpdate);
 171
 172         flushExistingIpRewrite();
 173         this.externalRouterMac = externalRouterMacUpdate;
 174         rebuildExistingIpRewrite();
 175     }
 176
 177     /**
 178      * Process the event.
 179      *
 180      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 181      * @param subnet An instance of NeutronSubnet object.
 182      */
 183     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 184         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 185     }
 186
 187     /**
 188      * Process the port event as a router interface event.
 189      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 190      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 191      *
 192      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 193      * @param neutronPort An instance of NeutronPort object.
 194      */
 195     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 196         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 197         if (!this.enabled) {
 198             return;
 199         }
 200
 201         final boolean isDelete = action == Action.DELETE;
 202
 203         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 204             if(!isDelete){
 205                 Node externalBridgeNode = getExternalBridgeNode();
 206                 if(externalBridgeNode != null){
 207                     LOG.info("Port {} is network router gateway interface, "
 208                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 209                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 210                 }else{
 211                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 212                 }
 213             }else{
 214                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 215
 216                 if(externalNetwork != null){
 217                     if(externalNetwork.isRouterExternal()){
 218                         final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 219                         if(externalSubnet != null){
 220                             gatewayMacResolver.stopPeriodicReferesh(new Ipv4Address(externalSubnet.getGatewayIP()));
 221                         }
 222                     }
 223                 }
 224             }
 225         }
 226
 227         // Treat the port event as a router interface event if the port belongs to router. This is a
 228         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 229         //
 230         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 231             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 232
 233             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 234                 NeutronRouter_Interface neutronRouterInterface =
 235                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 236                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 237                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 238
 239                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 240             }
 241         } else {
 242             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 243             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 244             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 245             // there.
 246             //
 247             if (!isDelete) {
 248                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 249                     NeutronRouter_Interface neutronRouterInterface =
 250                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 251                     if (neutronRouterInterface != null) {
 252                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 253                     }
 254                 }
 255             }
 256             this.updateL3ForNeutronPort(neutronPort, isDelete);
 257         }
 258     }
 259
 260     /**
 261      * Process the event.
 262      *
 263      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 264      * @param neutronRouter An instance of NeutronRouter object.
 265      */
 266     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 267         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 268     }
 269
 270     /**
 271      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 272      * delete the ports on the same subnet.
 273      *
 274      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 275      * @param neutronRouter An instance of NeutronRouter object.
 276      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 277      */
 278     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 279                                                   final NeutronRouter_Interface neutronRouterInterface,
 280                                                   Action action) {
 281         LOG.debug("Router interface {} got event {}. Subnet {}",
 282                      neutronRouterInterface.getPortUUID(),
 283                      action,
 284                      neutronRouterInterface.getSubnetUUID());
 285         if (!this.enabled) {
 286             return;
 287         }
 288
 289         final boolean isDelete = action == Action.DELETE;
 290
 291         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 292
 293         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 294         // see if they are affected by l3
 295         //
 296         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 297             boolean currPortShouldBeDeleted = false;
 298             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 299             if (isDelete) {
 300                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 301                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 302                         currPortShouldBeDeleted = true;
 303                         break;
 304                     }
 305                 }
 306             }
 307             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 308         }
 309     }
 310
 311     /**
 312      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 313      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 314      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 315      * way around) on OpenFlow Table 100.
 316      *
 317      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 318      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 319      */
 320     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 321                                              Action actionIn) {
 322         Preconditions.checkNotNull(neutronFloatingIP);
 323
 324         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 325                 neutronFloatingIP.getFixedIPAddress(),
 326                 neutronFloatingIP.getFloatingIPAddress(),
 327                 neutronFloatingIP.getFloatingNetworkUUID());
 328         if (!this.enabled) {
 329             return;
 330         }
 331
 332         Action action;
 333
 334         // Consider action to be delete if getFixedIPAddress is null
 335         //
 336         if (neutronFloatingIP.getFixedIPAddress() == null) {
 337             action = Action.DELETE;
 338         } else {
 339             action = actionIn;
 340         }
 341
 342         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 343
 344         if (action != Action.DELETE) {
 345             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 346
 347             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 348             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 349         } else {
 350             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 351             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 352
 353             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 354         }
 355     }
 356
 357     /**
 358      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 359      * ip, otherwise for newer one.
 360      */
 361     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 362         Preconditions.checkNotNull(neutronFloatingIP);
 363
 364         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 365         if (fid == null) {
 366             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 367                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 368             return;
 369         }
 370         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 371                                       action);
 372     }
 373
 374     /**
 375      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 376      * ip, otherwise for newer one.
 377      */
 378     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 379         Preconditions.checkNotNull(neutronFloatingIP);
 380
 381         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 382         if (fid == null) {
 383             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 384                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 385             return;
 386         }
 387         programOutboundIpRewriteStage1(fid, action);
 388     }
 389
 390     private void flushExistingIpRewrite() {
 391         for (FloatIpData fid : floatIpDataMapCache.values()) {
 392             programOutboundIpRewriteStage1(fid, Action.DELETE);
 393         }
 394     }
 395
 396     private void rebuildExistingIpRewrite() {
 397         for (FloatIpData fid : floatIpDataMapCache.values()) {
 398             programOutboundIpRewriteStage1(fid, Action.ADD);
 399         }
 400     }
 401
 402     /**
 403      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 404      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 405      * requests.
 406      */
 407     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 408         Preconditions.checkNotNull(neutronFloatingIP);
 409         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 410         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 411
 412         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 413             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 414                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 415             return;
 416         }
 417
 418         // find bridge Node where floating ip is configured by looking up cache for its port
 419         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 420         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 421         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 422         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 423         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 424         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 425
 426         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 427         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 428                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 429         final String providerSegmentationId = tenantNeutronNetwork != null ?
 430                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 431         final String neutronRouterMac = tenantNeutronNetwork != null ?
 432                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 433
 434         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 435                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 436                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 437                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 438             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 439                     fixedIpAddress,
 440                     floatingIpAddress,
 441                     neutronPortForFloatIp,
 442                     neutronTenantPortUuid,
 443                     providerSegmentationId,
 444                     floatingIpMac,
 445                     neutronRouterMac);
 446             return;
 447         }
 448
 449         // get ofport for patch port in br-int
 450         final Long dpId = nodeIfPair.getLeft();
 451         final Long ofPort = findOFPortForExtPatch(dpId);
 452         if (ofPort == null) {
 453             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 454                     dpId);
 455             return;
 456         }
 457
 458         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 459         //
 460         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 461                 Action.ADD)) {
 462             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 463                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 464             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 465             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 466                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 467                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 468         }
 469     }
 470
 471     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 472         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 473         if (floatIpData == null) {
 474             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 475             return;
 476         }
 477
 478         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 479                 floatIpData.floatingIpAddress, Action.DELETE)) {
 480             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 481             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 482                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 483         }
 484     }
 485
 486     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 487         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 488             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 489                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 490                 return neutronPort;
 491             }
 492         }
 493         return null;
 494     }
 495
 496     private Long findOFPortForExtPatch(Long dpId) {
 497         final String brInt = configurationService.getIntegrationBridgeName();
 498         final String brExt = configurationService.getExternalBridgeName();
 499         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 500
 501         Preconditions.checkNotNull(dpId);
 502         Preconditions.checkNotNull(portNameInt);
 503
 504         final long dpidPrimitive = dpId;
 505         for (Node node : nodeCacheManager.getBridgeNodes()) {
 506             if (dpidPrimitive == southbound.getDataPathId(node)) {
 507                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 508                         southbound.getTerminationPointOfBridge(node, portNameInt);
 509                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 510             }
 511         }
 512         return null;
 513     }
 514
 515     /**
 516      * Process the event.
 517      *
 518      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 519      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 520      */
 521     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 522         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 523     }
 524
 525     //
 526     // Callbacks from OVSDB's southbound handler
 527     //
 528     /**
 529      * Process the event.
 530      *
 531      * @param bridgeNode An instance of Node object.
 532      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 533      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 534      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 535      * object.
 536      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 537      */
 538     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 539                                      final NeutronNetwork neutronNetwork, Action action) {
 540         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 541                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 542         if (!this.enabled) {
 543             return;
 544         }
 545
 546         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 547         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 548         final Uuid interfaceUuid = intf.getInterfaceUuid();
 549
 550         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 551                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 552
 553         if (neutronPort != null) {
 554             final String neutronPortUuid = neutronPort.getPortUUID();
 555
 556             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 557                     dpId != null && interfaceUuid != null) {
 558                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 559             }
 560
 561             handleNeutronPortEvent(neutronPort, action);
 562         }
 563
 564         if (action == Action.DELETE && interfaceUuid != null) {
 565             handleInterfaceEventDelete(intf, dpId);
 566         }
 567     }
 568
 569     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 570         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 571         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 572                 neutronPortUuid, dpId, interfaceUuid.getValue());
 573     }
 574
 575     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 576         // Remove entry from neutronPortToDpIdCache based on interface uuid
 577         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 578             final String currPortUuid = entry.getKey();
 579             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 580                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 581                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 582                 neutronPortToDpIdCache.remove(currPortUuid);
 583                 break;
 584             }
 585         }
 586     }
 587
 588     //
 589     // Internal helpers
 590     //
 591     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 592
 593         final String networkUUID = neutronPort.getNetworkUUID();
 594         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 595
 596         // If there is no router interface handling the networkUUID, we are done
 597         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 598             return;
 599         }
 600
 601         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 602         // router interface are handled via handleNeutronRouterInterfaceEvent.
 603         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 604             return;
 605         }
 606
 607         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 608         final String providerSegmentationId = neutronNetwork != null ?
 609                                               neutronNetwork.getProviderSegmentationID() : null;
 610         final String tenantMac = neutronPort.getMacAddress();
 611
 612         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 613             tenantMac == null || tenantMac.isEmpty()) {
 614             // done: go no further w/out all the info needed...
 615             return;
 616         }
 617
 618         final Action action = isDelete ? Action.DELETE : Action.ADD;
 619         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 620         if (nodes.isEmpty()) {
 621             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 622         }
 623         for (Node node : nodes) {
 624             final Long dpid = getDpidForIntegrationBridge(node);
 625             if (dpid == null) {
 626                 continue;
 627             }
 628
 629             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 630                 final String tenantIpStr = neutronIP.getIpAddress();
 631                 if (tenantIpStr.isEmpty()) {
 632                     continue;
 633                 }
 634
 635                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 636                 // still needed when routing to subnets non-local to node (bug 2076).
 637                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 638
 639                 // Configure distributed ARP responder
 640                 if (flgDistributedARPEnabled) {
 641                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 642                 }
 643             }
 644         }
 645     }
 646
 647     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 648                                            String macAddress, String ipStr,
 649                                            Action actionForNode) {
 650         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 651         // will look at desired action for node.
 652
 653         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 654         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 655
 656         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 657             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 658                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 659             return;
 660         }
 661         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 662             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 663                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 664             return;
 665         }
 666
 667         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 668                                                        macAddress, ipStr, actionForNode);
 669         if (status.isSuccess()) {
 670             // Update cache
 671             if (actionForNode == Action.ADD) {
 672                 l3ForwardingCache.add(cacheKey);
 673             } else {
 674                 l3ForwardingCache.remove(cacheKey);
 675             }
 676         }
 677     }
 678
 679     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 680                                              String macAddress,
 681                                              String address,
 682                                              Action actionForNode) {
 683         Status status;
 684         try {
 685             InetAddress inetAddress = InetAddress.getByName(address);
 686             status = l3ForwardingProvider == null ?
 687                      new Status(StatusCode.SUCCESS) :
 688                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 689                                                                       inetAddress, macAddress, actionForNode);
 690         } catch (UnknownHostException e) {
 691             status = new Status(StatusCode.BADREQUEST);
 692         }
 693
 694         if (status.isSuccess()) {
 695             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 696                          l3ForwardingProvider == null ? "skipped" : "programmed",
 697                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 698         } else {
 699             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 700                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 701         }
 702         return status;
 703     }
 704
 705     // --
 706
 707     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 708                                                        Boolean isDelete) {
 709         Preconditions.checkNotNull(destNeutronRouterInterface);
 710
 711         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 712         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 713         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 714         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 715         final NeutronNetwork neutronNetwork = subnet != null ?
 716                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 717         final String destinationSegmentationId = neutronNetwork != null ?
 718                                                  neutronNetwork.getProviderSegmentationID() : null;
 719         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 720         final String cidr = subnet != null ? subnet.getCidr() : null;
 721         final int mask = getMaskLenFromCidr(cidr);
 722
 723         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 724                      destNeutronRouterInterface, isDelete);
 725
 726         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 727         // the local cache
 728         if (neutronNetwork != null) {
 729             if (macAddress == null || macAddress.isEmpty()) {
 730                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 731             }
 732             if (ipList == null || ipList.isEmpty()) {
 733                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 734             }
 735         }
 736
 737         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 738             cidr == null || cidr.isEmpty() ||
 739             macAddress == null || macAddress.isEmpty() ||
 740             ipList == null || ipList.isEmpty()) {
 741             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 742                          destinationSegmentationId, cidr, macAddress, ipList);
 743             // done: go no further w/out all the info needed...
 744             return;
 745         }
 746
 747         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 748
 749         // Keep cache for finding router's mac from network uuid -- add
 750         //
 751         if (! isDelete) {
 752             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 753             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 754             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 755         }
 756
 757         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 758         if (nodes.isEmpty()) {
 759             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 760         }
 761         for (Node node : nodes) {
 762             final Long dpid = getDpidForIntegrationBridge(node);
 763             if (dpid == null) {
 764                 continue;
 765             }
 766
 767             for (Neutron_IPs neutronIP : ipList) {
 768                 final String ipStr = neutronIP.getIpAddress();
 769                 if (ipStr.isEmpty()) {
 770                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 771                             node.getNodeId().getValue(), ipStr);
 772                     continue;
 773                 }
 774
 775                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 776                 //
 777                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 778                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 779                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 780                                                               neutronNetwork, destinationSegmentationId,
 781                                                               macAddress, ipStr, mask, actionForNode,
 782                                                               true /*isReflexsive*/);
 783                 }
 784
 785                 if (! isExternal) {
 786                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 787                             actionForNode);
 788                 }
 789                 // Enable ARP responder by default, because router interface needs to be responded always.
 790                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 791             }
 792
 793             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 794             // for the external neutron networks.
 795             //
 796             {
 797                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 798                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 799             }
 800         }
 801
 802         // Keep cache for finding router's mac from network uuid -- remove
 803         //
 804         if (isDelete) {
 805             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 806             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 807             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 808         }
 809     }
 810
 811     private void programFlowForNetworkFromExternal(final Node node,
 812                                                    final Long dpid,
 813                                                    final String destinationSegmentationId,
 814                                                    final String dstMacAddress,
 815                                                    final String destIpStr,
 816                                                    final int destMask,
 817                                                    final Action actionForNode) {
 818         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 819                 dstMacAddress, destIpStr, destMask, actionForNode);
 820     }
 821
 822     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 823                                                            final Long dpid,
 824                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 825                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 826                                                            final NeutronNetwork dstNeutronNetwork,
 827                                                            final String destinationSegmentationId,
 828                                                            final String dstMacAddress,
 829                                                            final String destIpStr,
 830                                                            final int destMask,
 831                                                            final Action actionForNode,
 832                                                            Boolean isReflexsive) {
 833         Preconditions.checkNotNull(srcNeutronRouterInterface);
 834         Preconditions.checkNotNull(dstNeutronRouterInterface);
 835
 836         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 837         if (sourceSubnetId == null) {
 838             LOG.error("Could not get provider Subnet ID from router interface {}",
 839                          srcNeutronRouterInterface.getID());
 840             return;
 841         }
 842
 843         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 844         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 845         if (sourceNetworkId == null) {
 846             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 847             return;
 848         }
 849
 850         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 851         if (sourceNetwork == null) {
 852             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 853             return;
 854         }
 855
 856         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 857             // Isolate subnets from different tenants within the same router
 858             return;
 859         }
 860         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 861         if (sourceSegmentationId == null) {
 862             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 863             return;
 864         }
 865         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 866             // Skip 'self'
 867             return;
 868         }
 869
 870         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 871                                      dstMacAddress, destIpStr, destMask, actionForNode);
 872
 873         // Flip roles src->dst; dst->src
 874         if (isReflexsive) {
 875             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 876             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 877             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 878             final String cidr2 = sourceSubnet.getCidr();
 879             final int mask2 = getMaskLenFromCidr(cidr2);
 880
 881             if (cidr2 == null || cidr2.isEmpty() ||
 882                 macAddress2 == null || macAddress2.isEmpty() ||
 883                 ipList2 == null || ipList2.isEmpty()) {
 884                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 885                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 886                 // done: go no further w/out all the info needed...
 887                 return;
 888             }
 889
 890             for (Neutron_IPs neutronIP2 : ipList2) {
 891                 final String ipStr2 = neutronIP2.getIpAddress();
 892                 if (ipStr2.isEmpty()) {
 893                     continue;
 894                 }
 895                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 896                                                           srcNeutronRouterInterface,
 897                                                           sourceNetwork, sourceSegmentationId,
 898                                                           macAddress2, ipStr2, mask2, actionForNode,
 899                                                           false /*isReflexsive*/);
 900             }
 901         }
 902     }
 903
 904     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 905                                               String destinationSegmentationId,
 906                                               String macAddress, String ipStr, int mask,
 907                                               Action actionForNode) {
 908         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 909         // will look at desired action for node.
 910         //
 911         final String cacheKey = node.getNodeId().getValue() + ":" +
 912                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 913                                 ipStr + "/" + Integer.toString(mask);
 914         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 915
 916         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 917             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 918                          " action {} is already done",
 919                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 920                          macAddress, ipStr, mask, actionForNode);
 921             return;
 922         }
 923         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 924             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 925                          " action {} is already done",
 926                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 927                          macAddress, ipStr, mask, actionForNode);
 928             return;
 929         }
 930
 931         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 932                                                           macAddress, ipStr, mask, actionForNode);
 933         if (status.isSuccess()) {
 934             // Update cache
 935             if (actionForNode == Action.ADD) {
 936                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 937                 routerInterfacesCache.add(cacheKey);
 938             } else {
 939                 // TODO: multiTenantAwareRouter.removeInterface(...);
 940                 routerInterfacesCache.remove(cacheKey);
 941             }
 942         }
 943     }
 944
 945     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 946                                                 String destinationSegmentationId,
 947                                                 String macAddress,
 948                                                 String address, int mask,
 949                                                 Action actionForNode) {
 950         Status status;
 951         try {
 952             InetAddress inetAddress = InetAddress.getByName(address);
 953             status = routingProvider == null ?
 954                      new Status(StatusCode.SUCCESS) :
 955                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 956                                                             macAddress, inetAddress, mask, actionForNode);
 957         } catch (UnknownHostException e) {
 958             status = new Status(StatusCode.BADREQUEST);
 959         }
 960
 961         if (status.isSuccess()) {
 962             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 963                          routingProvider == null ? "skipped" : "programmed",
 964                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 965                          actionForNode);
 966         } else {
 967             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 968                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 969                          actionForNode, status);
 970         }
 971         return status;
 972     }
 973
 974     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 975                                            String macAddress, String ipStr,
 976                                            Action action) {
 977         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 978         // will look at desired action for node.
 979         //
 980         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 981         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 982
 983         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 984             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 985                     dpid, segOrOfPort, macAddress, ipStr, action);
 986             return true;
 987         }
 988         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 989             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 990                     dpid, segOrOfPort, macAddress, ipStr, action);
 991             return true;
 992         }
 993
 994         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 995         if (status.isSuccess()) {
 996             // Update cache
 997             if (action == Action.ADD) {
 998                 staticArpEntryCache.add(cacheKey);
 999             } else {
1000                 staticArpEntryCache.remove(cacheKey);
1001             }
1002             return true;
1003         }
1004         return false;
1005     }
1006
1007     private Status programStaticArpStage2(Long dpid,
1008                                           String segOrOfPort,
1009                                           String macAddress,
1010                                           String address,
1011                                           Action action) {
1012         Status status;
1013         try {
1014             InetAddress inetAddress = InetAddress.getByName(address);
1015             status = arpProvider == null ?
1016                      new Status(StatusCode.SUCCESS) :
1017                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1018                                                        macAddress, inetAddress, action);
1019         } catch (UnknownHostException e) {
1020             status = new Status(StatusCode.BADREQUEST);
1021         }
1022
1023         if (status.isSuccess()) {
1024             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1025                          arpProvider == null ? "skipped" : "programmed",
1026                          macAddress, address, dpid, segOrOfPort, action);
1027         } else {
1028             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1029                          macAddress, address, dpid, segOrOfPort, action, status);
1030         }
1031         return status;
1032     }
1033
1034     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1035                                                   String matchAddress, String rewriteAddress,
1036                                                   Action action) {
1037         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1038         // will look at desired action for node.
1039         //
1040         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1041         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1042
1043         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1044             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1045                     " action {} is already done",
1046                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1047             return true;
1048         }
1049         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1050             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1051                     " action is already done",
1052                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1053             return true;
1054         }
1055
1056         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1057                 rewriteAddress, action);
1058         if (status.isSuccess()) {
1059             // Update cache
1060             if (action == Action.ADD) {
1061                 inboundIpRewriteCache.add(cacheKey);
1062             } else {
1063                 inboundIpRewriteCache.remove(cacheKey);
1064             }
1065             return true;
1066         }
1067         return false;
1068     }
1069
1070     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1071                                                  String matchAddress, String rewriteAddress,
1072                                                  Action action) {
1073         Status status;
1074         try {
1075             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1076             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1077             status = inboundNatProvider == null ?
1078                     new Status(StatusCode.SUCCESS) :
1079                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1080                             inetMatchAddress, inetRewriteAddress,
1081                             action);
1082         } catch (UnknownHostException e) {
1083             status = new Status(StatusCode.BADREQUEST);
1084         }
1085
1086         if (status.isSuccess()) {
1087             final boolean isSkipped = inboundNatProvider == null;
1088             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1089                     isSkipped ? "skipped" : "programmed",
1090                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1091         } else {
1092             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1093                          " status:{}",
1094                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1095                     status);
1096         }
1097         return status;
1098     }
1099
1100     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1101                                                  Action actionForRewriteExclusion) {
1102         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1103         // will look at desired action for node.
1104         //
1105         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1106         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1107
1108         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1109             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1110                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1111             return;
1112         }
1113         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1114             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1115                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1116             return;
1117         }
1118
1119         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1120                                                              actionForRewriteExclusion);
1121         if (status.isSuccess()) {
1122             // Update cache
1123             if (actionForRewriteExclusion == Action.ADD) {
1124                     outboundIpRewriteExclusionCache.add(cacheKey);
1125             } else {
1126                     outboundIpRewriteExclusionCache.remove(cacheKey);
1127             }
1128         }
1129     }
1130
1131     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1132                                                    Action actionForNode) {
1133         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1134                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1135
1136         if (status.isSuccess()) {
1137             final boolean isSkipped = outboundNatProvider == null;
1138             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1139                          isSkipped ? "skipped" : "programmed",
1140                          cidr, node.getNodeId().getValue(), actionForNode);
1141         } else {
1142             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1143                          cidr, node.getNodeId().getValue(), actionForNode, status);
1144         }
1145         return status;
1146     }
1147
1148     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1149         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1150         // will look at desired action for node.
1151         //
1152         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1153         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1154
1155         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1156             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1157                          "is already done",
1158                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1159             return;
1160         }
1161         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1162             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1163                          "is already done",
1164                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1165             return;
1166         }
1167
1168         Status status = this.programOutboundIpRewriteStage2(fid, action);
1169         if (status.isSuccess()) {
1170             // Update cache
1171             if (action == Action.ADD) {
1172                 outboundIpRewriteCache.add(cacheKey);
1173             } else {
1174                 outboundIpRewriteCache.remove(cacheKey);
1175             }
1176         }
1177     }
1178
1179     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1180         Status status;
1181         try {
1182             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1183             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1184             status = outboundNatProvider == null ?
1185                     new Status(StatusCode.SUCCESS) :
1186                     outboundNatProvider.programIpRewriteRule(
1187                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1188                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1189         } catch (UnknownHostException e) {
1190             status = new Status(StatusCode.BADREQUEST);
1191         }
1192
1193         if (status.isSuccess()) {
1194             final boolean isSkipped = outboundNatProvider == null;
1195             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1196                             " action {}",
1197                          isSkipped ? "skipped" : "programmed",
1198                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1199         } else {
1200             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1201                          " action {} status:{}",
1202                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1203         }
1204         return status;
1205     }
1206
1207     private int getMaskLenFromCidr(String cidr) {
1208         if (cidr == null) {
1209             return 0;
1210         }
1211         String[] splits = cidr.split("/");
1212         if (splits.length != 2) {
1213             return 0;
1214         }
1215
1216         int result;
1217         try {
1218             result = Integer.parseInt(splits[1].trim());
1219         } catch (NumberFormatException nfe) {
1220             result = 0;
1221         }
1222         return result;
1223     }
1224
1225     private Long getDpidForIntegrationBridge(Node node) {
1226         // Check if node is integration bridge; and only then return its dpid
1227         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1228             return southbound.getDataPathId(node);
1229         }
1230         return null;
1231     }
1232
1233     private Long getDpidForExternalBridge(Node node) {
1234         // Check if node is integration bridge; and only then return its dpid
1235         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1236             return southbound.getDataPathId(node);
1237         }
1238         return null;
1239     }
1240
1241     private Node getExternalBridgeNode(){
1242         //Pickup the first node that has external bridge (br-ex).
1243         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1244         //the external network is reachable from every br-ex
1245         // TODO: Consider other deployment scenario, and thing of better solution.
1246         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1247         for(Node node : allBridges){
1248             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1249                 return node;
1250             }
1251         }
1252         return null;
1253     }
1254
1255     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1256         NeutronSubnet extSubnet = null;
1257         for (NeutronSubnet subnet : neutronSubnetCache.getAllSubnets()){
1258             if(subnet.getPortsInSubnet().contains(gatewayPort)){
1259                 extSubnet = subnet;
1260                 break;
1261             }
1262         }
1263         return extSubnet;
1264     }
1265
1266     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1267
1268         Preconditions.checkNotNull(node);
1269         Preconditions.checkNotNull(gatewayPort);
1270         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1271
1272         if(externalNetwork != null){
1273             if(externalNetwork.isRouterExternal()){
1274                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1275                 if(externalSubnet != null){
1276                     if(externalSubnet.getGatewayIP() != null){
1277                         LOG.info("Trigger MAC resolution for gateway ip {} on Node {}", externalSubnet.getGatewayIP(), node.getNodeId());
1278
1279                         ListenableFuture<MacAddress> gatewayMacAddress =
1280                                 gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1281                                         new Ipv4Address(externalSubnet.getGatewayIP()),
1282                                         new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1283                                         new MacAddress(gatewayPort.getMacAddress()),
1284                                         false);
1285                         if(gatewayMacAddress != null){
1286                             Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1287                                 @Override
1288                                 public void onSuccess(MacAddress result) {
1289                                     if(result != null){
1290                                         updateExternalRouterMac(result.getValue());
1291                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(), result.getValue());
1292                                     }else{
1293                                         LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1294                                     }
1295                                 }
1296
1297                                 @Override
1298                                 public void onFailure(Throwable t) {
1299                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1300                                 }
1301                             }, gatewayMacResolverPool);
1302                         }
1303                     }else{
1304                         LOG.warn("No gateway IP address found for external subnet {}", externalSubnet);
1305                     }
1306                 }else{
1307                     LOG.warn("Neutron subnet not found for external network {}", externalNetwork);
1308                 }
1309             }
1310         }else{
1311             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1312         }
1313     }
1314
1315     /**
1316      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1317      *
1318      * @param ofPort the OF port number
1319      * @return the string with encoded OF port (example format "OFPort|999")
1320      */
1321     public static String encodeExcplicitOFPort(Long ofPort) {
1322         return "OFPort|" + ofPort.toString();
1323     }
1324
1325     @Override
1326     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1327         tenantNetworkManager =
1328                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1329         configurationService =
1330                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1331         arpProvider =
1332                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1333         inboundNatProvider =
1334                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1335         outboundNatProvider =
1336                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1337         routingProvider =
1338                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1339         l3ForwardingProvider =
1340                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1341         nodeCacheManager =
1342                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1343         southbound =
1344                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1345         gatewayMacResolver =
1346                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1347         initL3AdapterMembers();
1348     }
1349
1350     @Override
1351     public void setDependencies(Object impl) {
1352         if (impl instanceof INeutronNetworkCRUD) {
1353             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1354         } else if (impl instanceof INeutronPortCRUD) {
1355             neutronPortCache = (INeutronPortCRUD)impl;
1356         } else if (impl instanceof INeutronSubnetCRUD) {
1357             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1358         } else if (impl instanceof ArpProvider) {
1359             arpProvider = (ArpProvider)impl;
1360         } else if (impl instanceof InboundNatProvider) {
1361             inboundNatProvider = (InboundNatProvider)impl;
1362         } else if (impl instanceof OutboundNatProvider) {
1363             outboundNatProvider = (OutboundNatProvider)impl;
1364         } else if (impl instanceof RoutingProvider) {
1365             routingProvider = (RoutingProvider)impl;
1366         } else if (impl instanceof L3ForwardingProvider) {
1367             l3ForwardingProvider = (L3ForwardingProvider)impl;
1368         }else if (impl instanceof GatewayMacResolver) {
1369             gatewayMacResolver = (GatewayMacResolver)impl;
1370         }
1371     }
1372 }