openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import com.google.common.util.concurrent.FutureCallback;
  34 import com.google.common.util.concurrent.Futures;
  35 import com.google.common.util.concurrent.ListenableFuture;
  36
  37 import org.osgi.framework.BundleContext;
  38 import org.osgi.framework.ServiceReference;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 import java.net.InetAddress;
  43 import java.net.UnknownHostException;
  44 import java.util.HashMap;
  45 import java.util.HashSet;
  46 import java.util.ArrayList;
  47 import java.util.List;
  48 import java.util.Map;
  49 import java.util.Set;
  50 import java.util.concurrent.ExecutorService;
  51 import java.util.concurrent.Executors;
  52
  53 /**
  54  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  55  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  56  * as well as the multi-tenant router forwarding provider.
  57  */
  58 public class NeutronL3Adapter implements ConfigInterface {
  59     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  60
  61     // The implementation for each of these services is resolved by the OSGi Service Manager
  62     private volatile ConfigurationService configurationService;
  63     private volatile TenantNetworkManager tenantNetworkManager;
  64     private volatile NodeCacheManager nodeCacheManager;
  65     private volatile INeutronNetworkCRUD neutronNetworkCache;
  66     private volatile INeutronSubnetCRUD neutronSubnetCache;
  67     private volatile INeutronPortCRUD neutronPortCache;
  68     private volatile L3ForwardingProvider l3ForwardingProvider;
  69     private volatile InboundNatProvider inboundNatProvider;
  70     private volatile OutboundNatProvider outboundNatProvider;
  71     private volatile ArpProvider arpProvider;
  72     private volatile RoutingProvider routingProvider;
  73     private volatile GatewayMacResolver gatewayMacResolver;
  74
  75     private class FloatIpData {
  76         // br-int of node where floating ip is associated with tenant port
  77         private final Long dpid;
  78         // patch port in br-int used to reach br-ex
  79         private final Long ofPort;
  80         // segmentation id of the net where fixed ip is instantiated
  81         private final String segId;
  82         // mac address assigned to neutron port of floating ip
  83         private final String macAddress;
  84         private final String floatingIpAddress;
  85         // ip address given to tenant vm
  86         private final String fixedIpAddress;
  87         private final String neutronRouterMac;
  88
  89         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  90                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  91             this.dpid = dpid;
  92             this.ofPort = ofPort;
  93             this.segId = segId;
  94             this.macAddress = macAddress;
  95             this.floatingIpAddress = floatingIpAddress;
  96             this.fixedIpAddress = fixedIpAddress;
  97             this.neutronRouterMac = neutronRouterMac;
  98         }
  99     }
 100
 101     private Set<String> inboundIpRewriteCache;
 102     private Set<String> outboundIpRewriteCache;
 103     private Set<String> outboundIpRewriteExclusionCache;
 104     private Set<String> routerInterfacesCache;
 105     private Set<String> staticArpEntryCache;
 106     private Set<String> l3ForwardingCache;
 107     private Map<String, String> networkIdToRouterMacCache;
 108     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 109     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 110     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 111     private Map<String, FloatIpData> floatIpDataMapCache;
 112     private String externalRouterMac;
 113     private Boolean enabled = false;
 114     private Boolean flgDistributedARPEnabled = true;
 115     private Southbound southbound;
 116     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 117
 118     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 119     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 120     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 121     private static final String OWNER_FLOATING_IP = "network:floatingip";
 122     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 123
 124     public NeutronL3Adapter() {
 125         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 126     }
 127
 128     private void initL3AdapterMembers() {
 129         Preconditions.checkNotNull(configurationService);
 130
 131         if (configurationService.isL3ForwardingEnabled()) {
 132             this.inboundIpRewriteCache = new HashSet<>();
 133             this.outboundIpRewriteCache = new HashSet<>();
 134             this.outboundIpRewriteExclusionCache = new HashSet<>();
 135             this.routerInterfacesCache = new HashSet<>();
 136             this.staticArpEntryCache = new HashSet<>();
 137             this.l3ForwardingCache = new HashSet<>();
 138             this.networkIdToRouterMacCache = new HashMap<>();
 139             this.networkIdToRouterIpListCache = new HashMap<>();
 140             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 141             this.neutronPortToDpIdCache = new HashMap<>();
 142             this.floatIpDataMapCache = new HashMap<>();
 143
 144             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 145             if (this.externalRouterMac == null) {
 146                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 147             }
 148
 149             this.enabled = true;
 150             LOG.info("OVSDB L3 forwarding is enabled");
 151             if (configurationService.isDistributedArpDisabled()) {
 152                 this.flgDistributedARPEnabled = false;
 153                 LOG.info("Distributed ARP responder is disabled");
 154             } else {
 155                 LOG.debug("Distributed ARP responder is enabled");
 156             }
 157         } else {
 158             LOG.debug("OVSDB L3 forwarding is disabled");
 159         }
 160     }
 161
 162     //
 163     // Callbacks from OVSDB's northbound handlers
 164     //
 165
 166     /**
 167      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 168      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 169      * subscribe so that the mac address associated to the gateway ip address is available.
 170      *
 171      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 172      */
 173     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 174         Preconditions.checkNotNull(externalRouterMacUpdate);
 175
 176         flushExistingIpRewrite();
 177         this.externalRouterMac = externalRouterMacUpdate;
 178         rebuildExistingIpRewrite();
 179     }
 180
 181     /**
 182      * Process the event.
 183      *
 184      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 185      * @param subnet An instance of NeutronSubnet object.
 186      */
 187     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 188         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 189     }
 190
 191     /**
 192      * Process the port event as a router interface event.
 193      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 194      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 195      *
 196      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 197      * @param neutronPort An instance of NeutronPort object.
 198      */
 199     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 200         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 201         if (!this.enabled) {
 202             return;
 203         }
 204
 205         final boolean isDelete = action == Action.DELETE;
 206
 207         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 208             if(!isDelete){
 209                 Node externalBridgeNode = getExternalBridgeNode();
 210                 if(externalBridgeNode != null){
 211                     LOG.info("Port {} is network router gateway interface, "
 212                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 213                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 214                 }else{
 215                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 216                 }
 217             }else{
 218                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 219
 220                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 221                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 222                     // TODO support IPv6
 223                     if (externalSubnet != null &&
 224                             externalSubnet.getIpVersion() == 4) {
 225                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 226                     }
 227                 }
 228             }
 229         }
 230
 231         // Treat the port event as a router interface event if the port belongs to router. This is a
 232         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 233         //
 234         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 235             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 236
 237             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 238                 NeutronRouter_Interface neutronRouterInterface =
 239                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 240                 // id of router interface to be same as subnet
 241                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 242                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 243
 244                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 245             }
 246         } else {
 247             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 248             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 249             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 250             // there.
 251             //
 252             if (!isDelete && neutronPort.getFixedIPs() != null) {
 253                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 254                     NeutronRouter_Interface neutronRouterInterface =
 255                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 256                     if (neutronRouterInterface != null) {
 257                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 258                     }
 259                 }
 260             }
 261             this.updateL3ForNeutronPort(neutronPort, isDelete);
 262         }
 263     }
 264
 265     /**
 266      * Process the event.
 267      *
 268      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 269      * @param neutronRouter An instance of NeutronRouter object.
 270      */
 271     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 272         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 273     }
 274
 275     /**
 276      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 277      * delete the ports on the same subnet.
 278      *
 279      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 280      * @param neutronRouter An instance of NeutronRouter object.
 281      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 282      */
 283     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 284                                                   final NeutronRouter_Interface neutronRouterInterface,
 285                                                   Action action) {
 286         LOG.debug("Router interface {} got event {}. Subnet {}",
 287                      neutronRouterInterface.getPortUUID(),
 288                      action,
 289                      neutronRouterInterface.getSubnetUUID());
 290         if (!this.enabled) {
 291             return;
 292         }
 293
 294         final boolean isDelete = action == Action.DELETE;
 295
 296         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 297
 298         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 299         // see if they are affected by l3
 300         //
 301         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 302             boolean currPortShouldBeDeleted = false;
 303             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 304             if (isDelete) {
 305                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 306                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 307                         currPortShouldBeDeleted = true;
 308                         break;
 309                     }
 310                 }
 311             }
 312             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 313         }
 314
 315         if (isDelete) {
 316             /*
 317              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 318              */
 319             this.cleanupRouterCache(neutronRouterInterface);
 320         }
 321     }
 322
 323     /**
 324      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 325      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 326      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 327      * way around) on OpenFlow Table 100.
 328      *
 329      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 330      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 331      */
 332     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 333                                              Action actionIn) {
 334         Preconditions.checkNotNull(neutronFloatingIP);
 335
 336         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 337                 neutronFloatingIP.getFixedIPAddress(),
 338                 neutronFloatingIP.getFloatingIPAddress(),
 339                 neutronFloatingIP.getFloatingNetworkUUID());
 340         if (!this.enabled) {
 341             return;
 342         }
 343
 344         Action action;
 345
 346         // Consider action to be delete if getFixedIPAddress is null
 347         //
 348         if (neutronFloatingIP.getFixedIPAddress() == null) {
 349             action = Action.DELETE;
 350         } else {
 351             action = actionIn;
 352         }
 353
 354         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 355
 356         if (action != Action.DELETE) {
 357             // must be first, as it updates floatIpDataMapCache
 358             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 359
 360             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 361             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 362         } else {
 363             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 364             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 365
 366             // must be last, as it updates floatIpDataMapCache
 367             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 368         }
 369     }
 370
 371     /**
 372      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 373      * ip, otherwise for newer one.
 374      */
 375     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 376         Preconditions.checkNotNull(neutronFloatingIP);
 377
 378         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 379         if (fid == null) {
 380             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 381                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 382             return;
 383         }
 384         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 385                                       action);
 386     }
 387
 388     /**
 389      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 390      * ip, otherwise for newer one.
 391      */
 392     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 393         Preconditions.checkNotNull(neutronFloatingIP);
 394
 395         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 396         if (fid == null) {
 397             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 398                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 399             return;
 400         }
 401         programOutboundIpRewriteStage1(fid, action);
 402     }
 403
 404     private void flushExistingIpRewrite() {
 405         for (FloatIpData fid : floatIpDataMapCache.values()) {
 406             programOutboundIpRewriteStage1(fid, Action.DELETE);
 407         }
 408     }
 409
 410     private void rebuildExistingIpRewrite() {
 411         for (FloatIpData fid : floatIpDataMapCache.values()) {
 412             programOutboundIpRewriteStage1(fid, Action.ADD);
 413         }
 414     }
 415
 416     /**
 417      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 418      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 419      * requests.
 420      */
 421     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 422         Preconditions.checkNotNull(neutronFloatingIP);
 423         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 424         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 425
 426         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 427             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 428                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 429             return;
 430         }
 431
 432         // find bridge Node where floating ip is configured by looking up cache for its port
 433         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 434         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 435         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 436         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 437         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 438         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 439
 440         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 441         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 442                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 443         final String providerSegmentationId = tenantNeutronNetwork != null ?
 444                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 445         final String neutronRouterMac = tenantNeutronNetwork != null ?
 446                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 447
 448         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 449                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 450                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 451                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 452             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 453                     fixedIpAddress,
 454                     floatingIpAddress,
 455                     neutronPortForFloatIp,
 456                     neutronTenantPortUuid,
 457                     providerSegmentationId,
 458                     floatingIpMac,
 459                     neutronRouterMac);
 460             return;
 461         }
 462
 463         // get ofport for patch port in br-int
 464         final Long dpId = nodeIfPair.getLeft();
 465         final Long ofPort = findOFPortForExtPatch(dpId);
 466         if (ofPort == null) {
 467             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 468                     dpId);
 469             return;
 470         }
 471
 472         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 473         //
 474         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 475                 Action.ADD)) {
 476             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 477                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 478             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 479             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 480                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 481                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 482         }
 483     }
 484
 485     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 486         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 487         if (floatIpData == null) {
 488             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 489             return;
 490         }
 491
 492         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 493                 floatIpData.floatingIpAddress, Action.DELETE)) {
 494             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 495             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 496                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 497         }
 498     }
 499
 500     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 501         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 502             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 503                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 504                 return neutronPort;
 505             }
 506         }
 507         return null;
 508     }
 509
 510     private Long findOFPortForExtPatch(Long dpId) {
 511         final String brInt = configurationService.getIntegrationBridgeName();
 512         final String brExt = configurationService.getExternalBridgeName();
 513         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 514
 515         Preconditions.checkNotNull(dpId);
 516         Preconditions.checkNotNull(portNameInt);
 517
 518         final long dpidPrimitive = dpId;
 519         for (Node node : nodeCacheManager.getBridgeNodes()) {
 520             if (dpidPrimitive == southbound.getDataPathId(node)) {
 521                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 522                         southbound.getTerminationPointOfBridge(node, portNameInt);
 523                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 524             }
 525         }
 526         return null;
 527     }
 528
 529     /**
 530      * Process the event.
 531      *
 532      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 533      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 534      */
 535     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 536         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 537     }
 538
 539     //
 540     // Callbacks from OVSDB's southbound handler
 541     //
 542     /**
 543      * Process the event.
 544      *
 545      * @param bridgeNode An instance of Node object.
 546      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 547      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 548      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 549      * object.
 550      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 551      */
 552     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 553                                      final NeutronNetwork neutronNetwork, Action action) {
 554         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 555                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 556         if (!this.enabled) {
 557             return;
 558         }
 559
 560         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 561         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 562         final Uuid interfaceUuid = intf.getInterfaceUuid();
 563
 564         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 565                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 566
 567         if (neutronPort != null) {
 568             final String neutronPortUuid = neutronPort.getPortUUID();
 569
 570             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 571                     dpId != null && interfaceUuid != null) {
 572                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 573             }
 574
 575             handleNeutronPortEvent(neutronPort, action);
 576         }
 577
 578         if (action == Action.DELETE && interfaceUuid != null) {
 579             handleInterfaceEventDelete(intf, dpId);
 580         }
 581     }
 582
 583     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 584         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 585         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 586                 neutronPortUuid, dpId, interfaceUuid.getValue());
 587     }
 588
 589     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 590         // Remove entry from neutronPortToDpIdCache based on interface uuid
 591         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 592             final String currPortUuid = entry.getKey();
 593             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 594                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 595                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 596                 neutronPortToDpIdCache.remove(currPortUuid);
 597                 break;
 598             }
 599         }
 600     }
 601
 602     //
 603     // Internal helpers
 604     //
 605     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 606
 607         final String networkUUID = neutronPort.getNetworkUUID();
 608         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 609
 610         // If there is no router interface handling the networkUUID, we are done
 611         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 612             return;
 613         }
 614
 615         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 616         // router interface are handled via handleNeutronRouterInterfaceEvent.
 617         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 618             return;
 619         }
 620
 621         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 622         final String providerSegmentationId = neutronNetwork != null ?
 623                                               neutronNetwork.getProviderSegmentationID() : null;
 624         final String tenantMac = neutronPort.getMacAddress();
 625
 626         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 627             tenantMac == null || tenantMac.isEmpty()) {
 628             // done: go no further w/out all the info needed...
 629             return;
 630         }
 631
 632         final Action action = isDelete ? Action.DELETE : Action.ADD;
 633         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 634         if (nodes.isEmpty()) {
 635             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 636         }
 637         for (Node node : nodes) {
 638             final Long dpid = getDpidForIntegrationBridge(node);
 639             if (dpid == null) {
 640                 continue;
 641             }
 642
 643             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 644                 final String tenantIpStr = neutronIP.getIpAddress();
 645                 if (tenantIpStr.isEmpty()) {
 646                     continue;
 647                 }
 648
 649                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 650                 // still needed when routing to subnets non-local to node (bug 2076).
 651                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 652
 653                 // Configure distributed ARP responder
 654                 if (flgDistributedARPEnabled) {
 655                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 656                 }
 657             }
 658         }
 659     }
 660
 661     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 662                                            String macAddress, String ipStr,
 663                                            Action actionForNode) {
 664         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 665         // will look at desired action for node.
 666
 667         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 668         final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 669
 670         if (actionForNode == Action.DELETE && !isProgrammed) {
 671             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 672                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 673             return;
 674         }
 675         if (actionForNode == Action.ADD && isProgrammed) {
 676             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 677                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 678             return;
 679         }
 680
 681         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 682                                                        macAddress, ipStr, actionForNode);
 683         if (status.isSuccess()) {
 684             // Update cache
 685             if (actionForNode == Action.ADD) {
 686                 l3ForwardingCache.add(cacheKey);
 687             } else {
 688                 l3ForwardingCache.remove(cacheKey);
 689             }
 690         }
 691     }
 692
 693     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 694                                              String macAddress,
 695                                              String address,
 696                                              Action actionForNode) {
 697         Status status;
 698         try {
 699             InetAddress inetAddress = InetAddress.getByName(address);
 700             status = l3ForwardingProvider == null ?
 701                      new Status(StatusCode.SUCCESS) :
 702                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 703                                                                       inetAddress, macAddress, actionForNode);
 704         } catch (UnknownHostException e) {
 705             status = new Status(StatusCode.BADREQUEST);
 706         }
 707
 708         if (status.isSuccess()) {
 709             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 710                          l3ForwardingProvider == null ? "skipped" : "programmed",
 711                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 712         } else {
 713             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 714                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 715         }
 716         return status;
 717     }
 718
 719     // --
 720
 721     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 722                                                        Boolean isDelete) {
 723         Preconditions.checkNotNull(destNeutronRouterInterface);
 724
 725         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 726         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 727         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 728         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 729         final NeutronNetwork neutronNetwork = subnet != null ?
 730                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 731         final String destinationSegmentationId = neutronNetwork != null ?
 732                                                  neutronNetwork.getProviderSegmentationID() : null;
 733         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 734         final String cidr = subnet != null ? subnet.getCidr() : null;
 735         final int mask = getMaskLenFromCidr(cidr);
 736
 737         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 738                      destNeutronRouterInterface, isDelete);
 739
 740         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 741         // the local cache
 742         if (neutronNetwork != null) {
 743             if (macAddress == null || macAddress.isEmpty()) {
 744                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 745             }
 746             if (ipList == null || ipList.isEmpty()) {
 747                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 748             }
 749         }
 750
 751         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 752             cidr == null || cidr.isEmpty() ||
 753             macAddress == null || macAddress.isEmpty() ||
 754             ipList == null || ipList.isEmpty()) {
 755             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 756                          destinationSegmentationId, cidr, macAddress, ipList);
 757             // done: go no further w/out all the info needed...
 758             return;
 759         }
 760
 761         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 762
 763         // Keep cache for finding router's mac from network uuid -- add
 764         //
 765         if (! isDelete) {
 766             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 767             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 768             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 769         }
 770
 771         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 772         if (nodes.isEmpty()) {
 773             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 774         }
 775         for (Node node : nodes) {
 776             final Long dpid = getDpidForIntegrationBridge(node);
 777             if (dpid == null) {
 778                 continue;
 779             }
 780
 781             for (Neutron_IPs neutronIP : ipList) {
 782                 final String ipStr = neutronIP.getIpAddress();
 783                 if (ipStr.isEmpty()) {
 784                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 785                             node.getNodeId().getValue(), ipStr);
 786                     continue;
 787                 }
 788
 789                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 790                 //
 791                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 792                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 793                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 794                                                               neutronNetwork, destinationSegmentationId,
 795                                                               macAddress, ipStr, mask, actionForNode,
 796                                                               true /*isReflexsive*/);
 797                 }
 798
 799                 if (! isExternal) {
 800                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 801                             actionForNode);
 802                 }
 803                 // Enable ARP responder by default, because router interface needs to be responded always.
 804                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 805             }
 806
 807             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 808             // for the external neutron networks.
 809             //
 810             {
 811                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 812                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 813             }
 814         }
 815
 816         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
 817     }
 818
 819     private void programFlowForNetworkFromExternal(final Node node,
 820                                                    final Long dpid,
 821                                                    final String destinationSegmentationId,
 822                                                    final String dstMacAddress,
 823                                                    final String destIpStr,
 824                                                    final int destMask,
 825                                                    final Action actionForNode) {
 826         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 827                 dstMacAddress, destIpStr, destMask, actionForNode);
 828     }
 829
 830     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 831                                                            final Long dpid,
 832                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 833                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 834                                                            final NeutronNetwork dstNeutronNetwork,
 835                                                            final String destinationSegmentationId,
 836                                                            final String dstMacAddress,
 837                                                            final String destIpStr,
 838                                                            final int destMask,
 839                                                            final Action actionForNode,
 840                                                            Boolean isReflexsive) {
 841         Preconditions.checkNotNull(srcNeutronRouterInterface);
 842         Preconditions.checkNotNull(dstNeutronRouterInterface);
 843
 844         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 845         if (sourceSubnetId == null) {
 846             LOG.error("Could not get provider Subnet ID from router interface {}",
 847                          srcNeutronRouterInterface.getID());
 848             return;
 849         }
 850
 851         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 852         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 853         if (sourceNetworkId == null) {
 854             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 855             return;
 856         }
 857
 858         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 859         if (sourceNetwork == null) {
 860             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 861             return;
 862         }
 863
 864         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 865             // Isolate subnets from different tenants within the same router
 866             return;
 867         }
 868         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 869         if (sourceSegmentationId == null) {
 870             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 871             return;
 872         }
 873         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 874             // Skip 'self'
 875             return;
 876         }
 877
 878         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 879                                      dstMacAddress, destIpStr, destMask, actionForNode);
 880
 881         // Flip roles src->dst; dst->src
 882         if (isReflexsive) {
 883             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 884             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 885             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 886             final String cidr2 = sourceSubnet.getCidr();
 887             final int mask2 = getMaskLenFromCidr(cidr2);
 888
 889             if (cidr2 == null || cidr2.isEmpty() ||
 890                 macAddress2 == null || macAddress2.isEmpty() ||
 891                 ipList2 == null || ipList2.isEmpty()) {
 892                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 893                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 894                 // done: go no further w/out all the info needed...
 895                 return;
 896             }
 897
 898             for (Neutron_IPs neutronIP2 : ipList2) {
 899                 final String ipStr2 = neutronIP2.getIpAddress();
 900                 if (ipStr2.isEmpty()) {
 901                     continue;
 902                 }
 903                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 904                                                           srcNeutronRouterInterface,
 905                                                           sourceNetwork, sourceSegmentationId,
 906                                                           macAddress2, ipStr2, mask2, actionForNode,
 907                                                           false /*isReflexsive*/);
 908             }
 909         }
 910     }
 911
 912     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 913                                               String destinationSegmentationId,
 914                                               String macAddress, String ipStr, int mask,
 915                                               Action actionForNode) {
 916         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 917         // will look at desired action for node.
 918         //
 919         final String cacheKey = node.getNodeId().getValue() + ":" +
 920                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 921                                 ipStr + "/" + Integer.toString(mask);
 922         final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 923
 924         if (actionForNode == Action.DELETE && !isProgrammed) {
 925             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 926                          " action {} is already done",
 927                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 928                          macAddress, ipStr, mask, actionForNode);
 929             return;
 930         }
 931         if (actionForNode == Action.ADD && isProgrammed) {
 932             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 933                          " action {} is already done",
 934                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 935                          macAddress, ipStr, mask, actionForNode);
 936             return;
 937         }
 938
 939         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 940                                                           macAddress, ipStr, mask, actionForNode);
 941         if (status.isSuccess()) {
 942             // Update cache
 943             if (actionForNode == Action.ADD) {
 944                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 945                 routerInterfacesCache.add(cacheKey);
 946             } else {
 947                 // TODO: multiTenantAwareRouter.removeInterface(...);
 948                 routerInterfacesCache.remove(cacheKey);
 949             }
 950         }
 951     }
 952
 953     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 954                                                 String destinationSegmentationId,
 955                                                 String macAddress,
 956                                                 String address, int mask,
 957                                                 Action actionForNode) {
 958         Status status;
 959         try {
 960             InetAddress inetAddress = InetAddress.getByName(address);
 961             status = routingProvider == null ?
 962                      new Status(StatusCode.SUCCESS) :
 963                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 964                                                             macAddress, inetAddress, mask, actionForNode);
 965         } catch (UnknownHostException e) {
 966             status = new Status(StatusCode.BADREQUEST);
 967         }
 968
 969         if (status.isSuccess()) {
 970             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 971                          routingProvider == null ? "skipped" : "programmed",
 972                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 973                          actionForNode);
 974         } else {
 975             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 976                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 977                          actionForNode, status);
 978         }
 979         return status;
 980     }
 981
 982     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 983                                            String macAddress, String ipStr,
 984                                            Action action) {
 985         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 986         // will look at desired action for node.
 987         //
 988         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 989         final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 990
 991         if (action == Action.DELETE && !isProgrammed) {
 992             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 993                     dpid, segOrOfPort, macAddress, ipStr, action);
 994             return true;
 995         }
 996         if (action == Action.ADD && isProgrammed) {
 997             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 998                     dpid, segOrOfPort, macAddress, ipStr, action);
 999             return true;
1000         }
1001
1002         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1003         if (status.isSuccess()) {
1004             // Update cache
1005             if (action == Action.ADD) {
1006                 staticArpEntryCache.add(cacheKey);
1007             } else {
1008                 staticArpEntryCache.remove(cacheKey);
1009             }
1010             return true;
1011         }
1012         return false;
1013     }
1014
1015     private Status programStaticArpStage2(Long dpid,
1016                                           String segOrOfPort,
1017                                           String macAddress,
1018                                           String address,
1019                                           Action action) {
1020         Status status;
1021         try {
1022             InetAddress inetAddress = InetAddress.getByName(address);
1023             status = arpProvider == null ?
1024                      new Status(StatusCode.SUCCESS) :
1025                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1026                                                        macAddress, inetAddress, action);
1027         } catch (UnknownHostException e) {
1028             status = new Status(StatusCode.BADREQUEST);
1029         }
1030
1031         if (status.isSuccess()) {
1032             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1033                          arpProvider == null ? "skipped" : "programmed",
1034                          macAddress, address, dpid, segOrOfPort, action);
1035         } else {
1036             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1037                          macAddress, address, dpid, segOrOfPort, action, status);
1038         }
1039         return status;
1040     }
1041
1042     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1043                                                   String matchAddress, String rewriteAddress,
1044                                                   Action action) {
1045         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1046         // will look at desired action for node.
1047         //
1048         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1049         final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1050
1051         if (action == Action.DELETE && !isProgrammed) {
1052             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1053                     " action {} is already done",
1054                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1055             return true;
1056         }
1057         if (action == Action.ADD && isProgrammed) {
1058             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1059                     " action is already done",
1060                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1061             return true;
1062         }
1063
1064         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1065                 rewriteAddress, action);
1066         if (status.isSuccess()) {
1067             // Update cache
1068             if (action == Action.ADD) {
1069                 inboundIpRewriteCache.add(cacheKey);
1070             } else {
1071                 inboundIpRewriteCache.remove(cacheKey);
1072             }
1073             return true;
1074         }
1075         return false;
1076     }
1077
1078     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1079                                                  String matchAddress, String rewriteAddress,
1080                                                  Action action) {
1081         Status status;
1082         try {
1083             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1084             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1085             status = inboundNatProvider == null ?
1086                     new Status(StatusCode.SUCCESS) :
1087                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1088                             inetMatchAddress, inetRewriteAddress,
1089                             action);
1090         } catch (UnknownHostException e) {
1091             status = new Status(StatusCode.BADREQUEST);
1092         }
1093
1094         if (status.isSuccess()) {
1095             final boolean isSkipped = inboundNatProvider == null;
1096             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1097                     isSkipped ? "skipped" : "programmed",
1098                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1099         } else {
1100             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1101                          " status:{}",
1102                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1103                     status);
1104         }
1105         return status;
1106     }
1107
1108     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1109                                                  Action actionForRewriteExclusion) {
1110         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1111         // will look at desired action for node.
1112         //
1113         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1114         final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1115
1116         if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
1117             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1118                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1119             return;
1120         }
1121         if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
1122             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1123                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1124             return;
1125         }
1126
1127         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1128                                                              actionForRewriteExclusion);
1129         if (status.isSuccess()) {
1130             // Update cache
1131             if (actionForRewriteExclusion == Action.ADD) {
1132                     outboundIpRewriteExclusionCache.add(cacheKey);
1133             } else {
1134                     outboundIpRewriteExclusionCache.remove(cacheKey);
1135             }
1136         }
1137     }
1138
1139     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1140                                                    Action actionForNode) {
1141         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1142                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1143
1144         if (status.isSuccess()) {
1145             final boolean isSkipped = outboundNatProvider == null;
1146             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1147                          isSkipped ? "skipped" : "programmed",
1148                          cidr, node.getNodeId().getValue(), actionForNode);
1149         } else {
1150             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1151                          cidr, node.getNodeId().getValue(), actionForNode, status);
1152         }
1153         return status;
1154     }
1155
1156     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1157         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1158         // will look at desired action for node.
1159         //
1160         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1161         final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1162
1163         if (action == Action.DELETE && !isProgrammed) {
1164             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1165                          "is already done",
1166                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1167             return;
1168         }
1169         if (action == Action.ADD && isProgrammed) {
1170             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1171                          "is already done",
1172                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1173             return;
1174         }
1175
1176         Status status = this.programOutboundIpRewriteStage2(fid, action);
1177         if (status.isSuccess()) {
1178             // Update cache
1179             if (action == Action.ADD) {
1180                 outboundIpRewriteCache.add(cacheKey);
1181             } else {
1182                 outboundIpRewriteCache.remove(cacheKey);
1183             }
1184         }
1185     }
1186
1187     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1188         Status status;
1189         try {
1190             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1191             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1192             status = outboundNatProvider == null ?
1193                     new Status(StatusCode.SUCCESS) :
1194                     outboundNatProvider.programIpRewriteRule(
1195                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1196                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1197         } catch (UnknownHostException e) {
1198             status = new Status(StatusCode.BADREQUEST);
1199         }
1200
1201         if (status.isSuccess()) {
1202             final boolean isSkipped = outboundNatProvider == null;
1203             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1204                             " action {}",
1205                          isSkipped ? "skipped" : "programmed",
1206                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1207         } else {
1208             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1209                          " action {} status:{}",
1210                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1211         }
1212         return status;
1213     }
1214
1215     private int getMaskLenFromCidr(String cidr) {
1216         if (cidr == null) {
1217             return 0;
1218         }
1219         String[] splits = cidr.split("/");
1220         if (splits.length != 2) {
1221             return 0;
1222         }
1223
1224         int result;
1225         try {
1226             result = Integer.parseInt(splits[1].trim());
1227         } catch (NumberFormatException nfe) {
1228             result = 0;
1229         }
1230         return result;
1231     }
1232
1233     private Long getDpidForIntegrationBridge(Node node) {
1234         // Check if node is integration bridge; and only then return its dpid
1235         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1236             return southbound.getDataPathId(node);
1237         }
1238         return null;
1239     }
1240
1241     private Long getDpidForExternalBridge(Node node) {
1242         // Check if node is integration bridge; and only then return its dpid
1243         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1244             return southbound.getDataPathId(node);
1245         }
1246         return null;
1247     }
1248
1249     private Node getExternalBridgeNode(){
1250         //Pickup the first node that has external bridge (br-ex).
1251         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1252         //the external network is reachable from every br-ex
1253         // TODO: Consider other deployment scenario, and thing of better solution.
1254         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1255         for(Node node : allBridges){
1256             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1257                 return node;
1258             }
1259         }
1260         return null;
1261     }
1262
1263     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1264         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1265             String subnetUUID = neutronIPs.getSubnetUUID();
1266             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1267             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1268                 return extSubnet;
1269             }
1270             if (extSubnet == null) {
1271                 // TODO: when subnet is created, try again.
1272                 LOG.debug("subnet {} in not found", subnetUUID);
1273              }
1274         }
1275         return null;
1276     }
1277
1278      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1279          /*
1280           *  Fix for 4277
1281           *  Remove the router cache only after deleting the neutron
1282           *  port l3 flows.
1283           */
1284          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1285
1286          if (neutronPort != null) {
1287              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1288              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1289              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1290          }
1291      }
1292
1293     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1294
1295         Preconditions.checkNotNull(node);
1296         Preconditions.checkNotNull(gatewayPort);
1297         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1298
1299         if(externalNetwork != null){
1300             if(externalNetwork.isRouterExternal()){
1301                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1302
1303                 // TODO: address IPv6 case.
1304                 if (externalSubnet != null &&
1305                     externalSubnet.getIpVersion() == 4) {
1306                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1307                     ListenableFuture<MacAddress> gatewayMacAddress =
1308                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1309                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1310                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1311                                                              new MacAddress(gatewayPort.getMacAddress()),
1312                                                              true);
1313                     if(gatewayMacAddress != null){
1314                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1315                             @Override
1316                             public void onSuccess(MacAddress result) {
1317                                 if(result != null){
1318                                     if(!result.getValue().equals(externalRouterMac)){
1319                                         updateExternalRouterMac(result.getValue());
1320                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1321                                     }
1322                                 }else{
1323                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1324                                 }
1325                             }
1326
1327                             @Override
1328                             public void onFailure(Throwable t) {
1329                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1330                             }
1331                         }, gatewayMacResolverPool);
1332                     }
1333                 } else {
1334                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1335                 }
1336             }
1337         }else{
1338             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1339         }
1340     }
1341
1342     /**
1343      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1344      *
1345      * @param ofPort the OF port number
1346      * @return the string with encoded OF port (example format "OFPort|999")
1347      */
1348     public static String encodeExcplicitOFPort(Long ofPort) {
1349         return "OFPort|" + ofPort.toString();
1350     }
1351
1352     @Override
1353     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1354         tenantNetworkManager =
1355                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1356         configurationService =
1357                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1358         arpProvider =
1359                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1360         inboundNatProvider =
1361                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1362         outboundNatProvider =
1363                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1364         routingProvider =
1365                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1366         l3ForwardingProvider =
1367                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1368         nodeCacheManager =
1369                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1370         southbound =
1371                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1372         gatewayMacResolver =
1373                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1374         initL3AdapterMembers();
1375     }
1376
1377     @Override
1378     public void setDependencies(Object impl) {
1379         if (impl instanceof INeutronNetworkCRUD) {
1380             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1381         } else if (impl instanceof INeutronPortCRUD) {
1382             neutronPortCache = (INeutronPortCRUD)impl;
1383         } else if (impl instanceof INeutronSubnetCRUD) {
1384             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1385         } else if (impl instanceof ArpProvider) {
1386             arpProvider = (ArpProvider)impl;
1387         } else if (impl instanceof InboundNatProvider) {
1388             inboundNatProvider = (InboundNatProvider)impl;
1389         } else if (impl instanceof OutboundNatProvider) {
1390             outboundNatProvider = (OutboundNatProvider)impl;
1391         } else if (impl instanceof RoutingProvider) {
1392             routingProvider = (RoutingProvider)impl;
1393         } else if (impl instanceof L3ForwardingProvider) {
1394             l3ForwardingProvider = (L3ForwardingProvider)impl;
1395         }else if (impl instanceof GatewayMacResolver) {
1396             gatewayMacResolver = (GatewayMacResolver)impl;
1397         }
1398     }
1399 }