openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import com.google.common.util.concurrent.FutureCallback;
  34 import com.google.common.util.concurrent.Futures;
  35 import com.google.common.util.concurrent.ListenableFuture;
  36
  37 import org.osgi.framework.BundleContext;
  38 import org.osgi.framework.ServiceReference;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 import java.net.InetAddress;
  43 import java.net.UnknownHostException;
  44 import java.util.HashMap;
  45 import java.util.HashSet;
  46 import java.util.ArrayList;
  47 import java.util.List;
  48 import java.util.Map;
  49 import java.util.Set;
  50 import java.util.concurrent.ExecutorService;
  51 import java.util.concurrent.Executors;
  52
  53 /**
  54  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  55  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  56  * as well as the multi-tenant router forwarding provider.
  57  */
  58 public class NeutronL3Adapter implements ConfigInterface {
  59     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  60
  61     // The implementation for each of these services is resolved by the OSGi Service Manager
  62     private volatile ConfigurationService configurationService;
  63     private volatile TenantNetworkManager tenantNetworkManager;
  64     private volatile NodeCacheManager nodeCacheManager;
  65     private volatile INeutronNetworkCRUD neutronNetworkCache;
  66     private volatile INeutronSubnetCRUD neutronSubnetCache;
  67     private volatile INeutronPortCRUD neutronPortCache;
  68     private volatile L3ForwardingProvider l3ForwardingProvider;
  69     private volatile InboundNatProvider inboundNatProvider;
  70     private volatile OutboundNatProvider outboundNatProvider;
  71     private volatile ArpProvider arpProvider;
  72     private volatile RoutingProvider routingProvider;
  73     private volatile GatewayMacResolver gatewayMacResolver;
  74
  75     private class FloatIpData {
  76         // br-int of node where floating ip is associated with tenant port
  77         private final Long dpid;
  78         // patch port in br-int used to reach br-ex
  79         private final Long ofPort;
  80         // segmentation id of the net where fixed ip is instantiated
  81         private final String segId;
  82         // mac address assigned to neutron port of floating ip
  83         private final String macAddress;
  84         private final String floatingIpAddress;
  85         // ip address given to tenant vm
  86         private final String fixedIpAddress;
  87         private final String neutronRouterMac;
  88
  89         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  90                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  91             this.dpid = dpid;
  92             this.ofPort = ofPort;
  93             this.segId = segId;
  94             this.macAddress = macAddress;
  95             this.floatingIpAddress = floatingIpAddress;
  96             this.fixedIpAddress = fixedIpAddress;
  97             this.neutronRouterMac = neutronRouterMac;
  98         }
  99     }
 100
 101     private Set<String> inboundIpRewriteCache;
 102     private Set<String> outboundIpRewriteCache;
 103     private Set<String> outboundIpRewriteExclusionCache;
 104     private Set<String> routerInterfacesCache;
 105     private Set<String> staticArpEntryCache;
 106     private Set<String> l3ForwardingCache;
 107     private Map<String, String> networkIdToRouterMacCache;
 108     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 109     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 110     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 111     private Map<String, FloatIpData> floatIpDataMapCache;
 112     private String externalRouterMac;
 113     private Boolean enabled = false;
 114     private Boolean flgDistributedARPEnabled = true;
 115     private Southbound southbound;
 116     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 117
 118     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 119     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 120     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 121     private static final String OWNER_FLOATING_IP = "network:floatingip";
 122     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 123
 124     public NeutronL3Adapter() {
 125         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 126     }
 127
 128     private void initL3AdapterMembers() {
 129         Preconditions.checkNotNull(configurationService);
 130
 131         if (configurationService.isL3ForwardingEnabled()) {
 132             this.inboundIpRewriteCache = new HashSet<>();
 133             this.outboundIpRewriteCache = new HashSet<>();
 134             this.outboundIpRewriteExclusionCache = new HashSet<>();
 135             this.routerInterfacesCache = new HashSet<>();
 136             this.staticArpEntryCache = new HashSet<>();
 137             this.l3ForwardingCache = new HashSet<>();
 138             this.networkIdToRouterMacCache = new HashMap<>();
 139             this.networkIdToRouterIpListCache = new HashMap<>();
 140             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 141             this.neutronPortToDpIdCache = new HashMap<>();
 142             this.floatIpDataMapCache = new HashMap<>();
 143
 144             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 145             if (this.externalRouterMac == null) {
 146                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 147             }
 148
 149             this.enabled = true;
 150             LOG.info("OVSDB L3 forwarding is enabled");
 151             if (configurationService.isDistributedArpDisabled()) {
 152                 this.flgDistributedARPEnabled = false;
 153                 LOG.info("Distributed ARP responder is disabled");
 154             } else {
 155                 LOG.debug("Distributed ARP responder is enabled");
 156             }
 157         } else {
 158             LOG.debug("OVSDB L3 forwarding is disabled");
 159         }
 160     }
 161
 162     //
 163     // Callbacks from OVSDB's northbound handlers
 164     //
 165
 166     /**
 167      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 168      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 169      * subscribe so that the mac address associated to the gateway ip address is available.
 170      *
 171      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 172      */
 173     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 174         Preconditions.checkNotNull(externalRouterMacUpdate);
 175
 176         flushExistingIpRewrite();
 177         this.externalRouterMac = externalRouterMacUpdate;
 178         rebuildExistingIpRewrite();
 179     }
 180
 181     /**
 182      * Process the event.
 183      *
 184      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 185      * @param subnet An instance of NeutronSubnet object.
 186      */
 187     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 188         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 189     }
 190
 191     /**
 192      * Process the port event as a router interface event.
 193      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 194      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 195      *
 196      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 197      * @param neutronPort An instance of NeutronPort object.
 198      */
 199     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 200         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 201         if (!this.enabled) {
 202             return;
 203         }
 204
 205         final boolean isDelete = action == Action.DELETE;
 206
 207         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 208             if(!isDelete){
 209                 Node externalBridgeNode = getExternalBridgeNode();
 210                 if(externalBridgeNode != null){
 211                     LOG.info("Port {} is network router gateway interface, "
 212                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 213                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 214                 }else{
 215                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 216                 }
 217             }else{
 218                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 219
 220                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 221                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 222                     // TODO support IPv6
 223                     if (externalSubnet != null &&
 224                             externalSubnet.getIpVersion() == 4) {
 225                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 226                     }
 227                 }
 228             }
 229         }
 230
 231         // Treat the port event as a router interface event if the port belongs to router. This is a
 232         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 233         //
 234         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 235             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 236
 237             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 238                 NeutronRouter_Interface neutronRouterInterface =
 239                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 240                 // id of router interface to be same as subnet
 241                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 242                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 243
 244                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 245             }
 246         } else {
 247             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 248             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 249             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 250             // there.
 251             //
 252             if (!isDelete && neutronPort.getFixedIPs() != null) {
 253                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 254                     NeutronRouter_Interface neutronRouterInterface =
 255                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 256                     if (neutronRouterInterface != null) {
 257                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 258                     }
 259                 }
 260             }
 261             this.updateL3ForNeutronPort(neutronPort, isDelete);
 262         }
 263     }
 264
 265     /**
 266      * Process the event.
 267      *
 268      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 269      * @param neutronRouter An instance of NeutronRouter object.
 270      */
 271     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 272         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 273     }
 274
 275     /**
 276      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 277      * delete the ports on the same subnet.
 278      *
 279      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 280      * @param neutronRouter An instance of NeutronRouter object.
 281      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 282      */
 283     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 284                                                   final NeutronRouter_Interface neutronRouterInterface,
 285                                                   Action action) {
 286         LOG.debug("Router interface {} got event {}. Subnet {}",
 287                      neutronRouterInterface.getPortUUID(),
 288                      action,
 289                      neutronRouterInterface.getSubnetUUID());
 290         if (!this.enabled) {
 291             return;
 292         }
 293
 294         final boolean isDelete = action == Action.DELETE;
 295
 296         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 297
 298         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 299         // see if they are affected by l3
 300         //
 301         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 302             boolean currPortShouldBeDeleted = false;
 303             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 304             if (isDelete) {
 305                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 306                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 307                         currPortShouldBeDeleted = true;
 308                         break;
 309                     }
 310                 }
 311             }
 312             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 313         }
 314     }
 315
 316     /**
 317      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 318      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 319      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 320      * way around) on OpenFlow Table 100.
 321      *
 322      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 323      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 324      */
 325     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 326                                              Action actionIn) {
 327         Preconditions.checkNotNull(neutronFloatingIP);
 328
 329         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 330                 neutronFloatingIP.getFixedIPAddress(),
 331                 neutronFloatingIP.getFloatingIPAddress(),
 332                 neutronFloatingIP.getFloatingNetworkUUID());
 333         if (!this.enabled) {
 334             return;
 335         }
 336
 337         Action action;
 338
 339         // Consider action to be delete if getFixedIPAddress is null
 340         //
 341         if (neutronFloatingIP.getFixedIPAddress() == null) {
 342             action = Action.DELETE;
 343         } else {
 344             action = actionIn;
 345         }
 346
 347         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 348
 349         if (action != Action.DELETE) {
 350             // must be first, as it updates floatIpDataMapCache
 351             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 352
 353             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 354             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 355         } else {
 356             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 357             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 358
 359             // must be last, as it updates floatIpDataMapCache
 360             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 361         }
 362     }
 363
 364     /**
 365      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 366      * ip, otherwise for newer one.
 367      */
 368     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 369         Preconditions.checkNotNull(neutronFloatingIP);
 370
 371         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 372         if (fid == null) {
 373             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 374                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 375             return;
 376         }
 377         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 378                                       action);
 379     }
 380
 381     /**
 382      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 383      * ip, otherwise for newer one.
 384      */
 385     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 386         Preconditions.checkNotNull(neutronFloatingIP);
 387
 388         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 389         if (fid == null) {
 390             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 391                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 392             return;
 393         }
 394         programOutboundIpRewriteStage1(fid, action);
 395     }
 396
 397     private void flushExistingIpRewrite() {
 398         for (FloatIpData fid : floatIpDataMapCache.values()) {
 399             programOutboundIpRewriteStage1(fid, Action.DELETE);
 400         }
 401     }
 402
 403     private void rebuildExistingIpRewrite() {
 404         for (FloatIpData fid : floatIpDataMapCache.values()) {
 405             programOutboundIpRewriteStage1(fid, Action.ADD);
 406         }
 407     }
 408
 409     /**
 410      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 411      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 412      * requests.
 413      */
 414     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 415         Preconditions.checkNotNull(neutronFloatingIP);
 416         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 417         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 418
 419         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 420             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 421                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 422             return;
 423         }
 424
 425         // find bridge Node where floating ip is configured by looking up cache for its port
 426         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 427         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 428         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 429         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 430         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 431         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 432
 433         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 434         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 435                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 436         final String providerSegmentationId = tenantNeutronNetwork != null ?
 437                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 438         final String neutronRouterMac = tenantNeutronNetwork != null ?
 439                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 440
 441         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 442                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 443                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 444                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 445             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 446                     fixedIpAddress,
 447                     floatingIpAddress,
 448                     neutronPortForFloatIp,
 449                     neutronTenantPortUuid,
 450                     providerSegmentationId,
 451                     floatingIpMac,
 452                     neutronRouterMac);
 453             return;
 454         }
 455
 456         // get ofport for patch port in br-int
 457         final Long dpId = nodeIfPair.getLeft();
 458         final Long ofPort = findOFPortForExtPatch(dpId);
 459         if (ofPort == null) {
 460             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 461                     dpId);
 462             return;
 463         }
 464
 465         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 466         //
 467         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 468                 Action.ADD)) {
 469             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 470                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 471             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 472             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 473                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 474                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 475         }
 476     }
 477
 478     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 479         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 480         if (floatIpData == null) {
 481             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 482             return;
 483         }
 484
 485         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 486                 floatIpData.floatingIpAddress, Action.DELETE)) {
 487             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 488             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 489                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 490         }
 491     }
 492
 493     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 494         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 495             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 496                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 497                 return neutronPort;
 498             }
 499         }
 500         return null;
 501     }
 502
 503     private Long findOFPortForExtPatch(Long dpId) {
 504         final String brInt = configurationService.getIntegrationBridgeName();
 505         final String brExt = configurationService.getExternalBridgeName();
 506         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 507
 508         Preconditions.checkNotNull(dpId);
 509         Preconditions.checkNotNull(portNameInt);
 510
 511         final long dpidPrimitive = dpId;
 512         for (Node node : nodeCacheManager.getBridgeNodes()) {
 513             if (dpidPrimitive == southbound.getDataPathId(node)) {
 514                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 515                         southbound.getTerminationPointOfBridge(node, portNameInt);
 516                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 517             }
 518         }
 519         return null;
 520     }
 521
 522     /**
 523      * Process the event.
 524      *
 525      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 526      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 527      */
 528     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 529         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 530     }
 531
 532     //
 533     // Callbacks from OVSDB's southbound handler
 534     //
 535     /**
 536      * Process the event.
 537      *
 538      * @param bridgeNode An instance of Node object.
 539      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 540      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 541      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 542      * object.
 543      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 544      */
 545     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 546                                      final NeutronNetwork neutronNetwork, Action action) {
 547         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 548                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 549         if (!this.enabled) {
 550             return;
 551         }
 552
 553         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 554         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 555         final Uuid interfaceUuid = intf.getInterfaceUuid();
 556
 557         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 558                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 559
 560         if (neutronPort != null) {
 561             final String neutronPortUuid = neutronPort.getPortUUID();
 562
 563             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 564                     dpId != null && interfaceUuid != null) {
 565                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 566             }
 567
 568             handleNeutronPortEvent(neutronPort, action);
 569         }
 570
 571         if (action == Action.DELETE && interfaceUuid != null) {
 572             handleInterfaceEventDelete(intf, dpId);
 573         }
 574     }
 575
 576     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 577         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 578         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 579                 neutronPortUuid, dpId, interfaceUuid.getValue());
 580     }
 581
 582     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 583         // Remove entry from neutronPortToDpIdCache based on interface uuid
 584         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 585             final String currPortUuid = entry.getKey();
 586             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 587                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 588                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 589                 neutronPortToDpIdCache.remove(currPortUuid);
 590                 break;
 591             }
 592         }
 593     }
 594
 595     //
 596     // Internal helpers
 597     //
 598     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 599
 600         final String networkUUID = neutronPort.getNetworkUUID();
 601         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 602
 603         // If there is no router interface handling the networkUUID, we are done
 604         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 605             return;
 606         }
 607
 608         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 609         // router interface are handled via handleNeutronRouterInterfaceEvent.
 610         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 611             return;
 612         }
 613
 614         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 615         final String providerSegmentationId = neutronNetwork != null ?
 616                                               neutronNetwork.getProviderSegmentationID() : null;
 617         final String tenantMac = neutronPort.getMacAddress();
 618
 619         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 620             tenantMac == null || tenantMac.isEmpty()) {
 621             // done: go no further w/out all the info needed...
 622             return;
 623         }
 624
 625         final Action action = isDelete ? Action.DELETE : Action.ADD;
 626         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 627         if (nodes.isEmpty()) {
 628             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 629         }
 630         for (Node node : nodes) {
 631             final Long dpid = getDpidForIntegrationBridge(node);
 632             if (dpid == null) {
 633                 continue;
 634             }
 635
 636             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 637                 final String tenantIpStr = neutronIP.getIpAddress();
 638                 if (tenantIpStr.isEmpty()) {
 639                     continue;
 640                 }
 641
 642                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 643                 // still needed when routing to subnets non-local to node (bug 2076).
 644                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 645
 646                 // Configure distributed ARP responder
 647                 if (flgDistributedARPEnabled) {
 648                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 649                 }
 650             }
 651         }
 652     }
 653
 654     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 655                                            String macAddress, String ipStr,
 656                                            Action actionForNode) {
 657         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 658         // will look at desired action for node.
 659
 660         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 661         final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 662
 663         if (actionForNode == Action.DELETE && !isProgrammed) {
 664             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 665                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 666             return;
 667         }
 668         if (actionForNode == Action.ADD && isProgrammed) {
 669             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 670                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 671             return;
 672         }
 673
 674         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 675                                                        macAddress, ipStr, actionForNode);
 676         if (status.isSuccess()) {
 677             // Update cache
 678             if (actionForNode == Action.ADD) {
 679                 l3ForwardingCache.add(cacheKey);
 680             } else {
 681                 l3ForwardingCache.remove(cacheKey);
 682             }
 683         }
 684     }
 685
 686     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 687                                              String macAddress,
 688                                              String address,
 689                                              Action actionForNode) {
 690         Status status;
 691         try {
 692             InetAddress inetAddress = InetAddress.getByName(address);
 693             status = l3ForwardingProvider == null ?
 694                      new Status(StatusCode.SUCCESS) :
 695                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 696                                                                       inetAddress, macAddress, actionForNode);
 697         } catch (UnknownHostException e) {
 698             status = new Status(StatusCode.BADREQUEST);
 699         }
 700
 701         if (status.isSuccess()) {
 702             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 703                          l3ForwardingProvider == null ? "skipped" : "programmed",
 704                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 705         } else {
 706             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 707                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 708         }
 709         return status;
 710     }
 711
 712     // --
 713
 714     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 715                                                        Boolean isDelete) {
 716         Preconditions.checkNotNull(destNeutronRouterInterface);
 717
 718         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 719         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 720         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 721         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 722         final NeutronNetwork neutronNetwork = subnet != null ?
 723                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 724         final String destinationSegmentationId = neutronNetwork != null ?
 725                                                  neutronNetwork.getProviderSegmentationID() : null;
 726         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 727         final String cidr = subnet != null ? subnet.getCidr() : null;
 728         final int mask = getMaskLenFromCidr(cidr);
 729
 730         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 731                      destNeutronRouterInterface, isDelete);
 732
 733         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 734         // the local cache
 735         if (neutronNetwork != null) {
 736             if (macAddress == null || macAddress.isEmpty()) {
 737                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 738             }
 739             if (ipList == null || ipList.isEmpty()) {
 740                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 741             }
 742         }
 743
 744         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 745             cidr == null || cidr.isEmpty() ||
 746             macAddress == null || macAddress.isEmpty() ||
 747             ipList == null || ipList.isEmpty()) {
 748             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 749                          destinationSegmentationId, cidr, macAddress, ipList);
 750             // done: go no further w/out all the info needed...
 751             return;
 752         }
 753
 754         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 755
 756         // Keep cache for finding router's mac from network uuid -- add
 757         //
 758         if (! isDelete) {
 759             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 760             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 761             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 762         }
 763
 764         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 765         if (nodes.isEmpty()) {
 766             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 767         }
 768         for (Node node : nodes) {
 769             final Long dpid = getDpidForIntegrationBridge(node);
 770             if (dpid == null) {
 771                 continue;
 772             }
 773
 774             for (Neutron_IPs neutronIP : ipList) {
 775                 final String ipStr = neutronIP.getIpAddress();
 776                 if (ipStr.isEmpty()) {
 777                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 778                             node.getNodeId().getValue(), ipStr);
 779                     continue;
 780                 }
 781
 782                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 783                 //
 784                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 785                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 786                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 787                                                               neutronNetwork, destinationSegmentationId,
 788                                                               macAddress, ipStr, mask, actionForNode,
 789                                                               true /*isReflexsive*/);
 790                 }
 791
 792                 if (! isExternal) {
 793                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 794                             actionForNode);
 795                 }
 796                 // Enable ARP responder by default, because router interface needs to be responded always.
 797                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 798             }
 799
 800             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 801             // for the external neutron networks.
 802             //
 803             {
 804                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 805                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 806             }
 807         }
 808
 809         // Keep cache for finding router's mac from network uuid -- remove
 810         //
 811         if (isDelete) {
 812             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 813             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 814             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 815         }
 816     }
 817
 818     private void programFlowForNetworkFromExternal(final Node node,
 819                                                    final Long dpid,
 820                                                    final String destinationSegmentationId,
 821                                                    final String dstMacAddress,
 822                                                    final String destIpStr,
 823                                                    final int destMask,
 824                                                    final Action actionForNode) {
 825         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 826                 dstMacAddress, destIpStr, destMask, actionForNode);
 827     }
 828
 829     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 830                                                            final Long dpid,
 831                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 832                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 833                                                            final NeutronNetwork dstNeutronNetwork,
 834                                                            final String destinationSegmentationId,
 835                                                            final String dstMacAddress,
 836                                                            final String destIpStr,
 837                                                            final int destMask,
 838                                                            final Action actionForNode,
 839                                                            Boolean isReflexsive) {
 840         Preconditions.checkNotNull(srcNeutronRouterInterface);
 841         Preconditions.checkNotNull(dstNeutronRouterInterface);
 842
 843         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 844         if (sourceSubnetId == null) {
 845             LOG.error("Could not get provider Subnet ID from router interface {}",
 846                          srcNeutronRouterInterface.getID());
 847             return;
 848         }
 849
 850         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 851         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 852         if (sourceNetworkId == null) {
 853             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 854             return;
 855         }
 856
 857         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 858         if (sourceNetwork == null) {
 859             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 860             return;
 861         }
 862
 863         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 864             // Isolate subnets from different tenants within the same router
 865             return;
 866         }
 867         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 868         if (sourceSegmentationId == null) {
 869             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 870             return;
 871         }
 872         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 873             // Skip 'self'
 874             return;
 875         }
 876
 877         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 878                                      dstMacAddress, destIpStr, destMask, actionForNode);
 879
 880         // Flip roles src->dst; dst->src
 881         if (isReflexsive) {
 882             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 883             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 884             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 885             final String cidr2 = sourceSubnet.getCidr();
 886             final int mask2 = getMaskLenFromCidr(cidr2);
 887
 888             if (cidr2 == null || cidr2.isEmpty() ||
 889                 macAddress2 == null || macAddress2.isEmpty() ||
 890                 ipList2 == null || ipList2.isEmpty()) {
 891                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 892                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 893                 // done: go no further w/out all the info needed...
 894                 return;
 895             }
 896
 897             for (Neutron_IPs neutronIP2 : ipList2) {
 898                 final String ipStr2 = neutronIP2.getIpAddress();
 899                 if (ipStr2.isEmpty()) {
 900                     continue;
 901                 }
 902                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 903                                                           srcNeutronRouterInterface,
 904                                                           sourceNetwork, sourceSegmentationId,
 905                                                           macAddress2, ipStr2, mask2, actionForNode,
 906                                                           false /*isReflexsive*/);
 907             }
 908         }
 909     }
 910
 911     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 912                                               String destinationSegmentationId,
 913                                               String macAddress, String ipStr, int mask,
 914                                               Action actionForNode) {
 915         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 916         // will look at desired action for node.
 917         //
 918         final String cacheKey = node.getNodeId().getValue() + ":" +
 919                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 920                                 ipStr + "/" + Integer.toString(mask);
 921         final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 922
 923         if (actionForNode == Action.DELETE && !isProgrammed) {
 924             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 925                          " action {} is already done",
 926                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 927                          macAddress, ipStr, mask, actionForNode);
 928             return;
 929         }
 930         if (actionForNode == Action.ADD && isProgrammed) {
 931             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 932                          " action {} is already done",
 933                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 934                          macAddress, ipStr, mask, actionForNode);
 935             return;
 936         }
 937
 938         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 939                                                           macAddress, ipStr, mask, actionForNode);
 940         if (status.isSuccess()) {
 941             // Update cache
 942             if (actionForNode == Action.ADD) {
 943                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 944                 routerInterfacesCache.add(cacheKey);
 945             } else {
 946                 // TODO: multiTenantAwareRouter.removeInterface(...);
 947                 routerInterfacesCache.remove(cacheKey);
 948             }
 949         }
 950     }
 951
 952     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 953                                                 String destinationSegmentationId,
 954                                                 String macAddress,
 955                                                 String address, int mask,
 956                                                 Action actionForNode) {
 957         Status status;
 958         try {
 959             InetAddress inetAddress = InetAddress.getByName(address);
 960             status = routingProvider == null ?
 961                      new Status(StatusCode.SUCCESS) :
 962                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 963                                                             macAddress, inetAddress, mask, actionForNode);
 964         } catch (UnknownHostException e) {
 965             status = new Status(StatusCode.BADREQUEST);
 966         }
 967
 968         if (status.isSuccess()) {
 969             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 970                          routingProvider == null ? "skipped" : "programmed",
 971                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 972                          actionForNode);
 973         } else {
 974             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 975                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 976                          actionForNode, status);
 977         }
 978         return status;
 979     }
 980
 981     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 982                                            String macAddress, String ipStr,
 983                                            Action action) {
 984         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 985         // will look at desired action for node.
 986         //
 987         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 988         final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 989
 990         if (action == Action.DELETE && !isProgrammed) {
 991             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 992                     dpid, segOrOfPort, macAddress, ipStr, action);
 993             return true;
 994         }
 995         if (action == Action.ADD && isProgrammed) {
 996             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 997                     dpid, segOrOfPort, macAddress, ipStr, action);
 998             return true;
 999         }
1000
1001         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1002         if (status.isSuccess()) {
1003             // Update cache
1004             if (action == Action.ADD) {
1005                 staticArpEntryCache.add(cacheKey);
1006             } else {
1007                 staticArpEntryCache.remove(cacheKey);
1008             }
1009             return true;
1010         }
1011         return false;
1012     }
1013
1014     private Status programStaticArpStage2(Long dpid,
1015                                           String segOrOfPort,
1016                                           String macAddress,
1017                                           String address,
1018                                           Action action) {
1019         Status status;
1020         try {
1021             InetAddress inetAddress = InetAddress.getByName(address);
1022             status = arpProvider == null ?
1023                      new Status(StatusCode.SUCCESS) :
1024                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1025                                                        macAddress, inetAddress, action);
1026         } catch (UnknownHostException e) {
1027             status = new Status(StatusCode.BADREQUEST);
1028         }
1029
1030         if (status.isSuccess()) {
1031             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1032                          arpProvider == null ? "skipped" : "programmed",
1033                          macAddress, address, dpid, segOrOfPort, action);
1034         } else {
1035             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1036                          macAddress, address, dpid, segOrOfPort, action, status);
1037         }
1038         return status;
1039     }
1040
1041     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1042                                                   String matchAddress, String rewriteAddress,
1043                                                   Action action) {
1044         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1045         // will look at desired action for node.
1046         //
1047         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1048         final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1049
1050         if (action == Action.DELETE && !isProgrammed) {
1051             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1052                     " action {} is already done",
1053                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1054             return true;
1055         }
1056         if (action == Action.ADD && isProgrammed) {
1057             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1058                     " action is already done",
1059                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1060             return true;
1061         }
1062
1063         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1064                 rewriteAddress, action);
1065         if (status.isSuccess()) {
1066             // Update cache
1067             if (action == Action.ADD) {
1068                 inboundIpRewriteCache.add(cacheKey);
1069             } else {
1070                 inboundIpRewriteCache.remove(cacheKey);
1071             }
1072             return true;
1073         }
1074         return false;
1075     }
1076
1077     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1078                                                  String matchAddress, String rewriteAddress,
1079                                                  Action action) {
1080         Status status;
1081         try {
1082             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1083             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1084             status = inboundNatProvider == null ?
1085                     new Status(StatusCode.SUCCESS) :
1086                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1087                             inetMatchAddress, inetRewriteAddress,
1088                             action);
1089         } catch (UnknownHostException e) {
1090             status = new Status(StatusCode.BADREQUEST);
1091         }
1092
1093         if (status.isSuccess()) {
1094             final boolean isSkipped = inboundNatProvider == null;
1095             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1096                     isSkipped ? "skipped" : "programmed",
1097                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1098         } else {
1099             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1100                          " status:{}",
1101                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1102                     status);
1103         }
1104         return status;
1105     }
1106
1107     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1108                                                  Action actionForRewriteExclusion) {
1109         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1110         // will look at desired action for node.
1111         //
1112         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1113         final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1114
1115         if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
1116             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1117                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1118             return;
1119         }
1120         if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
1121             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1122                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1123             return;
1124         }
1125
1126         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1127                                                              actionForRewriteExclusion);
1128         if (status.isSuccess()) {
1129             // Update cache
1130             if (actionForRewriteExclusion == Action.ADD) {
1131                     outboundIpRewriteExclusionCache.add(cacheKey);
1132             } else {
1133                     outboundIpRewriteExclusionCache.remove(cacheKey);
1134             }
1135         }
1136     }
1137
1138     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1139                                                    Action actionForNode) {
1140         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1141                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1142
1143         if (status.isSuccess()) {
1144             final boolean isSkipped = outboundNatProvider == null;
1145             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1146                          isSkipped ? "skipped" : "programmed",
1147                          cidr, node.getNodeId().getValue(), actionForNode);
1148         } else {
1149             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1150                          cidr, node.getNodeId().getValue(), actionForNode, status);
1151         }
1152         return status;
1153     }
1154
1155     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1156         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1157         // will look at desired action for node.
1158         //
1159         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1160         final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1161
1162         if (action == Action.DELETE && !isProgrammed) {
1163             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1164                          "is already done",
1165                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1166             return;
1167         }
1168         if (action == Action.ADD && isProgrammed) {
1169             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1170                          "is already done",
1171                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1172             return;
1173         }
1174
1175         Status status = this.programOutboundIpRewriteStage2(fid, action);
1176         if (status.isSuccess()) {
1177             // Update cache
1178             if (action == Action.ADD) {
1179                 outboundIpRewriteCache.add(cacheKey);
1180             } else {
1181                 outboundIpRewriteCache.remove(cacheKey);
1182             }
1183         }
1184     }
1185
1186     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1187         Status status;
1188         try {
1189             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1190             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1191             status = outboundNatProvider == null ?
1192                     new Status(StatusCode.SUCCESS) :
1193                     outboundNatProvider.programIpRewriteRule(
1194                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1195                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1196         } catch (UnknownHostException e) {
1197             status = new Status(StatusCode.BADREQUEST);
1198         }
1199
1200         if (status.isSuccess()) {
1201             final boolean isSkipped = outboundNatProvider == null;
1202             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1203                             " action {}",
1204                          isSkipped ? "skipped" : "programmed",
1205                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1206         } else {
1207             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1208                          " action {} status:{}",
1209                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1210         }
1211         return status;
1212     }
1213
1214     private int getMaskLenFromCidr(String cidr) {
1215         if (cidr == null) {
1216             return 0;
1217         }
1218         String[] splits = cidr.split("/");
1219         if (splits.length != 2) {
1220             return 0;
1221         }
1222
1223         int result;
1224         try {
1225             result = Integer.parseInt(splits[1].trim());
1226         } catch (NumberFormatException nfe) {
1227             result = 0;
1228         }
1229         return result;
1230     }
1231
1232     private Long getDpidForIntegrationBridge(Node node) {
1233         // Check if node is integration bridge; and only then return its dpid
1234         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1235             return southbound.getDataPathId(node);
1236         }
1237         return null;
1238     }
1239
1240     private Long getDpidForExternalBridge(Node node) {
1241         // Check if node is integration bridge; and only then return its dpid
1242         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1243             return southbound.getDataPathId(node);
1244         }
1245         return null;
1246     }
1247
1248     private Node getExternalBridgeNode(){
1249         //Pickup the first node that has external bridge (br-ex).
1250         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1251         //the external network is reachable from every br-ex
1252         // TODO: Consider other deployment scenario, and thing of better solution.
1253         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1254         for(Node node : allBridges){
1255             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1256                 return node;
1257             }
1258         }
1259         return null;
1260     }
1261
1262     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1263         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1264             String subnetUUID = neutronIPs.getSubnetUUID();
1265             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1266             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1267                 return extSubnet;
1268             }
1269             if (extSubnet == null) {
1270                 // TODO: when subnet is created, try again.
1271                 LOG.debug("subnet {} in not found", subnetUUID);
1272              }
1273         }
1274         return null;
1275     }
1276
1277     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1278
1279         Preconditions.checkNotNull(node);
1280         Preconditions.checkNotNull(gatewayPort);
1281         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1282
1283         if(externalNetwork != null){
1284             if(externalNetwork.isRouterExternal()){
1285                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1286
1287                 // TODO: address IPv6 case.
1288                 if (externalSubnet != null &&
1289                     externalSubnet.getIpVersion() == 4) {
1290                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1291                     ListenableFuture<MacAddress> gatewayMacAddress =
1292                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1293                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1294                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1295                                                              new MacAddress(gatewayPort.getMacAddress()),
1296                                                              true);
1297                     if(gatewayMacAddress != null){
1298                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1299                             @Override
1300                             public void onSuccess(MacAddress result) {
1301                                 if(result != null){
1302                                     if(!result.getValue().equals(externalRouterMac)){
1303                                         updateExternalRouterMac(result.getValue());
1304                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1305                                     }
1306                                 }else{
1307                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1308                                 }
1309                             }
1310
1311                             @Override
1312                             public void onFailure(Throwable t) {
1313                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1314                             }
1315                         }, gatewayMacResolverPool);
1316                     }
1317                 } else {
1318                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1319                 }
1320             }
1321         }else{
1322             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1323         }
1324     }
1325
1326     /**
1327      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1328      *
1329      * @param ofPort the OF port number
1330      * @return the string with encoded OF port (example format "OFPort|999")
1331      */
1332     public static String encodeExcplicitOFPort(Long ofPort) {
1333         return "OFPort|" + ofPort.toString();
1334     }
1335
1336     @Override
1337     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1338         tenantNetworkManager =
1339                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1340         configurationService =
1341                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1342         arpProvider =
1343                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1344         inboundNatProvider =
1345                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1346         outboundNatProvider =
1347                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1348         routingProvider =
1349                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1350         l3ForwardingProvider =
1351                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1352         nodeCacheManager =
1353                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1354         southbound =
1355                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1356         gatewayMacResolver =
1357                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1358         initL3AdapterMembers();
1359     }
1360
1361     @Override
1362     public void setDependencies(Object impl) {
1363         if (impl instanceof INeutronNetworkCRUD) {
1364             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1365         } else if (impl instanceof INeutronPortCRUD) {
1366             neutronPortCache = (INeutronPortCRUD)impl;
1367         } else if (impl instanceof INeutronSubnetCRUD) {
1368             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1369         } else if (impl instanceof ArpProvider) {
1370             arpProvider = (ArpProvider)impl;
1371         } else if (impl instanceof InboundNatProvider) {
1372             inboundNatProvider = (InboundNatProvider)impl;
1373         } else if (impl instanceof OutboundNatProvider) {
1374             outboundNatProvider = (OutboundNatProvider)impl;
1375         } else if (impl instanceof RoutingProvider) {
1376             routingProvider = (RoutingProvider)impl;
1377         } else if (impl instanceof L3ForwardingProvider) {
1378             l3ForwardingProvider = (L3ForwardingProvider)impl;
1379         }else if (impl instanceof GatewayMacResolver) {
1380             gatewayMacResolver = (GatewayMacResolver)impl;
1381         }
1382     }
1383 }