openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  31 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  32
  33 import com.google.common.base.Preconditions;
  34 import com.google.common.util.concurrent.FutureCallback;
  35 import com.google.common.util.concurrent.Futures;
  36 import com.google.common.util.concurrent.ListenableFuture;
  37
  38 import org.osgi.framework.BundleContext;
  39 import org.osgi.framework.ServiceReference;
  40 import org.slf4j.Logger;
  41 import org.slf4j.LoggerFactory;
  42
  43 import java.net.InetAddress;
  44 import java.net.UnknownHostException;
  45 import java.util.HashMap;
  46 import java.util.HashSet;
  47 import java.util.ArrayList;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.Set;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53
  54 /**
  55  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  56  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  57  * as well as the multi-tenant router forwarding provider.
  58  */
  59 public class NeutronL3Adapter implements ConfigInterface {
  60     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  61
  62     // The implementation for each of these services is resolved by the OSGi Service Manager
  63     private volatile ConfigurationService configurationService;
  64     private volatile TenantNetworkManager tenantNetworkManager;
  65     private volatile NodeCacheManager nodeCacheManager;
  66     private volatile INeutronNetworkCRUD neutronNetworkCache;
  67     private volatile INeutronSubnetCRUD neutronSubnetCache;
  68     private volatile INeutronPortCRUD neutronPortCache;
  69     private volatile L3ForwardingProvider l3ForwardingProvider;
  70     private volatile InboundNatProvider inboundNatProvider;
  71     private volatile OutboundNatProvider outboundNatProvider;
  72     private volatile ArpProvider arpProvider;
  73     private volatile RoutingProvider routingProvider;
  74     private volatile GatewayMacResolver gatewayMacResolver;
  75
  76     private class FloatIpData {
  77         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  78         private final Long ofPort;        // patch port in br-int used to reach br-ex
  79         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  80         private final String macAddress;  // mac address assigned to neutron port of floating ip
  81         private final String floatingIpAddress;
  82         private final String fixedIpAddress;  // ip address given to tenant vm
  83         private final String neutronRouterMac;
  84
  85         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  86                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  87             this.dpid = dpid;
  88             this.ofPort = ofPort;
  89             this.segId = segId;
  90             this.macAddress = macAddress;
  91             this.floatingIpAddress = floatingIpAddress;
  92             this.fixedIpAddress = fixedIpAddress;
  93             this.neutronRouterMac = neutronRouterMac;
  94         }
  95     }
  96
  97     private Set<String> inboundIpRewriteCache;
  98     private Set<String> outboundIpRewriteCache;
  99     private Set<String> outboundIpRewriteExclusionCache;
 100     private Set<String> routerInterfacesCache;
 101     private Set<String> staticArpEntryCache;
 102     private Set<String> l3ForwardingCache;
 103     private Map<String, String> networkIdToRouterMacCache;
 104     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 105     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 106     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 107     private Map<String, FloatIpData> floatIpDataMapCache;
 108     private String externalRouterMac;
 109     private Boolean enabled = false;
 110     private Boolean flgDistributedARPEnabled = true;
 111     private Southbound southbound;
 112     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 113
 114     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 115     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 116     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 117     private static final String OWNER_FLOATING_IP = "network:floatingip";
 118     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 119
 120     public NeutronL3Adapter() {
 121         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 122     }
 123
 124     private void initL3AdapterMembers() {
 125         Preconditions.checkNotNull(configurationService);
 126
 127         if (configurationService.isL3ForwardingEnabled()) {
 128             this.inboundIpRewriteCache = new HashSet<>();
 129             this.outboundIpRewriteCache = new HashSet<>();
 130             this.outboundIpRewriteExclusionCache = new HashSet<>();
 131             this.routerInterfacesCache = new HashSet<>();
 132             this.staticArpEntryCache = new HashSet<>();
 133             this.l3ForwardingCache = new HashSet<>();
 134             this.networkIdToRouterMacCache = new HashMap<>();
 135             this.networkIdToRouterIpListCache = new HashMap<>();
 136             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 137             this.neutronPortToDpIdCache = new HashMap<>();
 138             this.floatIpDataMapCache = new HashMap<>();
 139
 140             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 141             if (this.externalRouterMac == null) {
 142                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 143             }
 144
 145             this.enabled = true;
 146             LOG.info("OVSDB L3 forwarding is enabled");
 147             if (configurationService.isDistributedArpDisabled()) {
 148                 this.flgDistributedARPEnabled = false;
 149                 LOG.info("Distributed ARP responder is disabled");
 150             } else {
 151                 LOG.debug("Distributed ARP responder is enabled");
 152             }
 153         } else {
 154             LOG.debug("OVSDB L3 forwarding is disabled");
 155         }
 156     }
 157
 158     //
 159     // Callbacks from OVSDB's northbound handlers
 160     //
 161
 162     /**
 163      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 164      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 165      * subscribe so that the mac address associated to the gateway ip address is available.
 166      *
 167      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 168      */
 169     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 170         Preconditions.checkNotNull(externalRouterMacUpdate);
 171
 172         flushExistingIpRewrite();
 173         this.externalRouterMac = externalRouterMacUpdate;
 174         rebuildExistingIpRewrite();
 175     }
 176
 177     /**
 178      * Process the event.
 179      *
 180      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 181      * @param subnet An instance of NeutronSubnet object.
 182      */
 183     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 184         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 185         if (!this.enabled) {
 186             return;
 187         }
 188     }
 189
 190     /**
 191      * Process the port event as a router interface event.
 192      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 193      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 194      *
 195      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 196      * @param neutronPort An instance of NeutronPort object.
 197      */
 198     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 199         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 200         if (!this.enabled) {
 201             return;
 202         }
 203
 204         final boolean isDelete = action == Action.DELETE;
 205
 206         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 207             if(!isDelete){
 208                 Node externalBridgeNode = getExternalBridgeNode();
 209                 if(externalBridgeNode != null){
 210                     LOG.info("Port {} is network router gateway interface, "
 211                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 212                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 213                 }else{
 214                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 215                 }
 216             }else{
 217                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 218
 219                 if(externalNetwork != null){
 220                     if(externalNetwork.isRouterExternal()){
 221                         final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 222                         if(externalSubnet != null){
 223                             gatewayMacResolver.stopPeriodicReferesh(new Ipv4Address(externalSubnet.getGatewayIP()));
 224                         }
 225                     }
 226                 }
 227             }
 228         }
 229
 230         // Treat the port event as a router interface event if the port belongs to router. This is a
 231         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 232         //
 233         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 234             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 235
 236             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 237                 NeutronRouter_Interface neutronRouterInterface =
 238                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 239                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 240                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 241
 242                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 243             }
 244         } else {
 245             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 246             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 247             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 248             // there.
 249             //
 250             if (!isDelete) {
 251                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 252                     NeutronRouter_Interface neutronRouterInterface =
 253                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 254                     if (neutronRouterInterface != null) {
 255                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 256                     }
 257                 }
 258             }
 259             this.updateL3ForNeutronPort(neutronPort, isDelete);
 260         }
 261     }
 262
 263     /**
 264      * Process the event.
 265      *
 266      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 267      * @param neutronRouter An instance of NeutronRouter object.
 268      */
 269     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 270         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 271         if (!this.enabled) {
 272             return;
 273         }
 274     }
 275
 276     /**
 277      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 278      * delete the ports on the same subnet.
 279      *
 280      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 281      * @param neutronRouter An instance of NeutronRouter object.
 282      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 283      */
 284     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 285                                                   final NeutronRouter_Interface neutronRouterInterface,
 286                                                   Action action) {
 287         LOG.debug("Router interface {} got event {}. Subnet {}",
 288                      neutronRouterInterface.getPortUUID(),
 289                      action,
 290                      neutronRouterInterface.getSubnetUUID());
 291         if (!this.enabled) {
 292             return;
 293         }
 294
 295         final boolean isDelete = action == Action.DELETE;
 296
 297         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 298
 299         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 300         // see if they are affected by l3
 301         //
 302         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 303             boolean currPortShouldBeDeleted = false;
 304             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 305             if (isDelete) {
 306                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 307                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 308                         currPortShouldBeDeleted = true;
 309                         break;
 310                     }
 311                 }
 312             }
 313             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 314         }
 315     }
 316
 317     /**
 318      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 319      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 320      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 321      * way around) on OpenFlow Table 100.
 322      *
 323      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 324      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 325      */
 326     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 327                                              Action actionIn) {
 328         Preconditions.checkNotNull(neutronFloatingIP);
 329
 330         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 331                 neutronFloatingIP.getFixedIPAddress(),
 332                 neutronFloatingIP.getFloatingIPAddress(),
 333                 neutronFloatingIP.getFloatingNetworkUUID());
 334         if (!this.enabled) {
 335             return;
 336         }
 337
 338         Action action;
 339
 340         // Consider action to be delete if getFixedIPAddress is null
 341         //
 342         if (neutronFloatingIP.getFixedIPAddress() == null) {
 343             action = Action.DELETE;
 344         } else {
 345             action = actionIn;
 346         }
 347
 348         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 349
 350         if (action != Action.DELETE) {
 351             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 352
 353             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 354             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 355         } else {
 356             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 357             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 358
 359             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 360         }
 361     }
 362
 363     /**
 364      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 365      * ip, otherwise for newer one.
 366      */
 367     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 368         Preconditions.checkNotNull(neutronFloatingIP);
 369
 370         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 371         if (fid == null) {
 372             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 373                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 374             return;
 375         }
 376         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 377                                       action);
 378     }
 379
 380     /**
 381      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 382      * ip, otherwise for newer one.
 383      */
 384     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 385         Preconditions.checkNotNull(neutronFloatingIP);
 386
 387         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 388         if (fid == null) {
 389             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 390                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 391             return;
 392         }
 393         programOutboundIpRewriteStage1(fid, action);
 394     }
 395
 396     private void flushExistingIpRewrite() {
 397         for (FloatIpData fid : floatIpDataMapCache.values()) {
 398             programOutboundIpRewriteStage1(fid, Action.DELETE);
 399         }
 400     }
 401
 402     private void rebuildExistingIpRewrite() {
 403         for (FloatIpData fid : floatIpDataMapCache.values()) {
 404             programOutboundIpRewriteStage1(fid, Action.ADD);
 405         }
 406     }
 407
 408     /**
 409      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 410      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 411      * requests.
 412      */
 413     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 414         Preconditions.checkNotNull(neutronFloatingIP);
 415         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 416         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 417
 418         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 419             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 420                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 421             return;
 422         }
 423
 424         // find bridge Node where floating ip is configured by looking up cache for its port
 425         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 426         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 427         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 428         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 429         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 430         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 431
 432         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 433         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 434                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 435         final String providerSegmentationId = tenantNeutronNetwork != null ?
 436                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 437         final String neutronRouterMac = tenantNeutronNetwork != null ?
 438                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 439
 440         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 441                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 442                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 443                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 444             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 445                     fixedIpAddress,
 446                     floatingIpAddress,
 447                     neutronPortForFloatIp,
 448                     neutronTenantPortUuid,
 449                     providerSegmentationId,
 450                     floatingIpMac,
 451                     neutronRouterMac);
 452             return;
 453         }
 454
 455         // get ofport for patch port in br-int
 456         final Long dpId = nodeIfPair.getLeft();
 457         final Long ofPort = findOFPortForExtPatch(dpId);
 458         if (ofPort == null) {
 459             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 460                     dpId);
 461             return;
 462         }
 463
 464         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 465         //
 466         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 467                 Action.ADD)) {
 468             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 469                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 470             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 471             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 472                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 473                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 474         }
 475     }
 476
 477     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 478         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 479         if (floatIpData == null) {
 480             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 481             return;
 482         }
 483
 484         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 485                 floatIpData.floatingIpAddress, Action.DELETE)) {
 486             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 487             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 488                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 489         }
 490     }
 491
 492     private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 493         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 494             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 495                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 496                 return neutronPort;
 497             }
 498         }
 499         return null;
 500     }
 501
 502     private final Long findOFPortForExtPatch(Long dpId) {
 503         final String brInt = configurationService.getIntegrationBridgeName();
 504         final String brExt = configurationService.getExternalBridgeName();
 505         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 506
 507         Preconditions.checkNotNull(dpId);
 508         Preconditions.checkNotNull(portNameInt);
 509
 510         final long dpidPrimitive = dpId.longValue();
 511         for (Node node : nodeCacheManager.getBridgeNodes()) {
 512             if (dpidPrimitive == southbound.getDataPathId(node)) {
 513                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 514                         southbound.getTerminationPointOfBridge(node, portNameInt);
 515                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 516             }
 517         }
 518         return null;
 519     }
 520
 521     /**
 522      * Process the event.
 523      *
 524      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 525      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 526      */
 527     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 528         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 529         if (!this.enabled) {
 530             return;
 531         }
 532     }
 533
 534     //
 535     // Callbacks from OVSDB's southbound handler
 536     //
 537     /**
 538      * Process the event.
 539      *
 540      * @param bridgeNode An instance of Node object.
 541      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 542      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 543      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 544      * object.
 545      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 546      */
 547     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 548                                      final NeutronNetwork neutronNetwork, Action action) {
 549         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 550                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 551         if (!this.enabled) {
 552             return;
 553         }
 554
 555         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 556         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 557         final Uuid interfaceUuid = intf.getInterfaceUuid();
 558
 559         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 560                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 561
 562         if (neutronPort != null) {
 563             final String neutronPortUuid = neutronPort.getPortUUID();
 564
 565             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 566                     dpId != null && interfaceUuid != null) {
 567                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 568             }
 569
 570             handleNeutronPortEvent(neutronPort, action);
 571         }
 572
 573         if (action == Action.DELETE && interfaceUuid != null) {
 574             handleInterfaceEventDelete(intf, dpId);
 575         }
 576     }
 577
 578     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 579         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 580         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 581                 neutronPortUuid, dpId, interfaceUuid.getValue());
 582     }
 583
 584     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 585         // Remove entry from neutronPortToDpIdCache based on interface uuid
 586         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 587             final String currPortUuid = entry.getKey();
 588             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 589                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 590                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 591                 neutronPortToDpIdCache.remove(currPortUuid);
 592                 break;
 593             }
 594         }
 595     }
 596
 597     //
 598     // Internal helpers
 599     //
 600     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 601
 602         final String networkUUID = neutronPort.getNetworkUUID();
 603         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 604
 605         // If there is no router interface handling the networkUUID, we are done
 606         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 607             return;
 608         }
 609
 610         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 611         // router interface are handled via handleNeutronRouterInterfaceEvent.
 612         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 613             return;
 614         }
 615
 616         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 617         final String providerSegmentationId = neutronNetwork != null ?
 618                                               neutronNetwork.getProviderSegmentationID() : null;
 619         final String tenantMac = neutronPort.getMacAddress();
 620
 621         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 622             tenantMac == null || tenantMac.isEmpty()) {
 623             // done: go no further w/out all the info needed...
 624             return;
 625         }
 626
 627         final Action action = isDelete ? Action.DELETE : Action.ADD;
 628         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 629         if (nodes.isEmpty()) {
 630             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 631         }
 632         for (Node node : nodes) {
 633             final Long dpid = getDpidForIntegrationBridge(node);
 634             if (dpid == null) {
 635                 continue;
 636             }
 637
 638             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 639                 final String tenantIpStr = neutronIP.getIpAddress();
 640                 if (tenantIpStr.isEmpty()) {
 641                     continue;
 642                 }
 643
 644                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 645                 // still needed when routing to subnets non-local to node (bug 2076).
 646                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 647
 648                 // Configure distributed ARP responder
 649                 if (true == flgDistributedARPEnabled) {
 650                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 651                 }
 652             }
 653         }
 654     }
 655
 656     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 657                                            String macAddress, String ipStr,
 658                                            Action actionForNode) {
 659         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 660         // will look at desired action for node.
 661
 662         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 663         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 664
 665         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 666             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 667                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 668             return;
 669         }
 670         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 671             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 672                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 673             return;
 674         }
 675
 676         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 677                                                        macAddress, ipStr, actionForNode);
 678         if (status.isSuccess()) {
 679             // Update cache
 680             if (actionForNode == Action.ADD) {
 681                 l3ForwardingCache.add(cacheKey);
 682             } else {
 683                 l3ForwardingCache.remove(cacheKey);
 684             }
 685         }
 686     }
 687
 688     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 689                                              String macAddress,
 690                                              String address,
 691                                              Action actionForNode) {
 692         Status status;
 693         try {
 694             InetAddress inetAddress = InetAddress.getByName(address);
 695             status = l3ForwardingProvider == null ?
 696                      new Status(StatusCode.SUCCESS) :
 697                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 698                                                                       inetAddress, macAddress, actionForNode);
 699         } catch (UnknownHostException e) {
 700             status = new Status(StatusCode.BADREQUEST);
 701         }
 702
 703         if (status.isSuccess()) {
 704             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 705                          l3ForwardingProvider == null ? "skipped" : "programmed",
 706                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 707         } else {
 708             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 709                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 710         }
 711         return status;
 712     }
 713
 714     // --
 715
 716     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 717                                                        Boolean isDelete) {
 718         Preconditions.checkNotNull(destNeutronRouterInterface);
 719
 720         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 721         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 722         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 723         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 724         final NeutronNetwork neutronNetwork = subnet != null ?
 725                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 726         final String destinationSegmentationId = neutronNetwork != null ?
 727                                                  neutronNetwork.getProviderSegmentationID() : null;
 728         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 729         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 730         final String cidr = subnet != null ? subnet.getCidr() : null;
 731         final int mask = getMaskLenFromCidr(cidr);
 732
 733         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 734                      destNeutronRouterInterface, isDelete);
 735
 736         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 737         // the local cache
 738         if (neutronNetwork != null) {
 739             if (macAddress == null || macAddress.isEmpty()) {
 740                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 741             }
 742             if (ipList == null || ipList.isEmpty()) {
 743                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 744             }
 745         }
 746
 747         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 748             cidr == null || cidr.isEmpty() ||
 749             macAddress == null || macAddress.isEmpty() ||
 750             ipList == null || ipList.isEmpty()) {
 751             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 752                          destinationSegmentationId, cidr, macAddress, ipList);
 753             // done: go no further w/out all the info needed...
 754             return;
 755         }
 756
 757         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 758
 759         // Keep cache for finding router's mac from network uuid -- add
 760         //
 761         if (! isDelete) {
 762             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 763             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 764             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 765         }
 766
 767         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 768         if (nodes.isEmpty()) {
 769             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 770         }
 771         for (Node node : nodes) {
 772             final Long dpid = getDpidForIntegrationBridge(node);
 773             if (dpid == null) {
 774                 continue;
 775             }
 776
 777             for (Neutron_IPs neutronIP : ipList) {
 778                 final String ipStr = neutronIP.getIpAddress();
 779                 if (ipStr.isEmpty()) {
 780                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 781                             node.getNodeId().getValue(), ipStr);
 782                     continue;
 783                 }
 784
 785                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 786                 //
 787                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 788                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 789                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 790                                                               neutronNetwork, destinationSegmentationId,
 791                                                               macAddress, ipStr, mask, actionForNode,
 792                                                               true /*isReflexsive*/);
 793                 }
 794
 795                 if (! isExternal) {
 796                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 797                             actionForNode);
 798                 }
 799                 // Enable ARP responder by default, because router interface needs to be responded always.
 800                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 801             }
 802
 803             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 804             // for the external neutron networks.
 805             //
 806             {
 807                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 808                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 809             }
 810         }
 811
 812         // Keep cache for finding router's mac from network uuid -- remove
 813         //
 814         if (isDelete) {
 815             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 816             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 817             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 818         }
 819     }
 820
 821     private void programFlowForNetworkFromExternal(final Node node,
 822                                                    final Long dpid,
 823                                                    final String destinationSegmentationId,
 824                                                    final String dstMacAddress,
 825                                                    final String destIpStr,
 826                                                    final int destMask,
 827                                                    final Action actionForNode) {
 828         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 829                 dstMacAddress, destIpStr, destMask, actionForNode);
 830     }
 831
 832     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 833                                                            final Long dpid,
 834                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 835                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 836                                                            final NeutronNetwork dstNeutronNetwork,
 837                                                            final String destinationSegmentationId,
 838                                                            final String dstMacAddress,
 839                                                            final String destIpStr,
 840                                                            final int destMask,
 841                                                            final Action actionForNode,
 842                                                            Boolean isReflexsive) {
 843         Preconditions.checkNotNull(srcNeutronRouterInterface);
 844         Preconditions.checkNotNull(dstNeutronRouterInterface);
 845
 846         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 847         if (sourceSubnetId == null) {
 848             LOG.error("Could not get provider Subnet ID from router interface {}",
 849                          srcNeutronRouterInterface.getID());
 850             return;
 851         }
 852
 853         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 854         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 855         if (sourceNetworkId == null) {
 856             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 857             return;
 858         }
 859
 860         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 861         if (sourceNetwork == null) {
 862             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 863             return;
 864         }
 865
 866         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 867             // Isolate subnets from different tenants within the same router
 868             return;
 869         }
 870         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 871         if (sourceSegmentationId == null) {
 872             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 873             return;
 874         }
 875         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 876             // Skip 'self'
 877             return;
 878         }
 879
 880         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 881                                      dstMacAddress, destIpStr, destMask, actionForNode);
 882
 883         // Flip roles src->dst; dst->src
 884         if (isReflexsive) {
 885             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 886             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 887             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 888             final String cidr2 = sourceSubnet.getCidr();
 889             final int mask2 = getMaskLenFromCidr(cidr2);
 890
 891             if (cidr2 == null || cidr2.isEmpty() ||
 892                 macAddress2 == null || macAddress2.isEmpty() ||
 893                 ipList2 == null || ipList2.isEmpty()) {
 894                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 895                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 896                 // done: go no further w/out all the info needed...
 897                 return;
 898             }
 899
 900             for (Neutron_IPs neutronIP2 : ipList2) {
 901                 final String ipStr2 = neutronIP2.getIpAddress();
 902                 if (ipStr2.isEmpty()) {
 903                     continue;
 904                 }
 905                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 906                                                           srcNeutronRouterInterface,
 907                                                           sourceNetwork, sourceSegmentationId,
 908                                                           macAddress2, ipStr2, mask2, actionForNode,
 909                                                           false /*isReflexsive*/);
 910             }
 911         }
 912     }
 913
 914     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 915                                               String destinationSegmentationId,
 916                                               String macAddress, String ipStr, int mask,
 917                                               Action actionForNode) {
 918         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 919         // will look at desired action for node.
 920         //
 921         final String cacheKey = node.getNodeId().getValue() + ":" +
 922                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 923                                 ipStr + "/" + Integer.toString(mask);
 924         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 925
 926         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 927             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 928                          " action {} is already done",
 929                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 930                          macAddress, ipStr, mask, actionForNode);
 931             return;
 932         }
 933         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 934             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 935                          " action {} is already done",
 936                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 937                          macAddress, ipStr, mask, actionForNode);
 938             return;
 939         }
 940
 941         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 942                                                           macAddress, ipStr, mask, actionForNode);
 943         if (status.isSuccess()) {
 944             // Update cache
 945             if (actionForNode == Action.ADD) {
 946                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 947                 routerInterfacesCache.add(cacheKey);
 948             } else {
 949                 // TODO: multiTenantAwareRouter.removeInterface(...);
 950                 routerInterfacesCache.remove(cacheKey);
 951             }
 952         }
 953     }
 954
 955     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 956                                                 String destinationSegmentationId,
 957                                                 String macAddress,
 958                                                 String address, int mask,
 959                                                 Action actionForNode) {
 960         Status status;
 961         try {
 962             InetAddress inetAddress = InetAddress.getByName(address);
 963             status = routingProvider == null ?
 964                      new Status(StatusCode.SUCCESS) :
 965                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 966                                                             macAddress, inetAddress, mask, actionForNode);
 967         } catch (UnknownHostException e) {
 968             status = new Status(StatusCode.BADREQUEST);
 969         }
 970
 971         if (status.isSuccess()) {
 972             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 973                          routingProvider == null ? "skipped" : "programmed",
 974                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 975                          actionForNode);
 976         } else {
 977             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 978                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 979                          actionForNode, status);
 980         }
 981         return status;
 982     }
 983
 984     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 985                                            String macAddress, String ipStr,
 986                                            Action action) {
 987         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 988         // will look at desired action for node.
 989         //
 990         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 991         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 992
 993         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 994             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 995                     dpid, segOrOfPort, macAddress, ipStr, action);
 996             return true;
 997         }
 998         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 999             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1000                     dpid, segOrOfPort, macAddress, ipStr, action);
1001             return true;
1002         }
1003
1004         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1005         if (status.isSuccess()) {
1006             // Update cache
1007             if (action == Action.ADD) {
1008                 staticArpEntryCache.add(cacheKey);
1009             } else {
1010                 staticArpEntryCache.remove(cacheKey);
1011             }
1012             return true;
1013         }
1014         return false;
1015     }
1016
1017     private Status programStaticArpStage2(Long dpid,
1018                                           String segOrOfPort,
1019                                           String macAddress,
1020                                           String address,
1021                                           Action action) {
1022         Status status;
1023         try {
1024             InetAddress inetAddress = InetAddress.getByName(address);
1025             status = arpProvider == null ?
1026                      new Status(StatusCode.SUCCESS) :
1027                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1028                                                        macAddress, inetAddress, action);
1029         } catch (UnknownHostException e) {
1030             status = new Status(StatusCode.BADREQUEST);
1031         }
1032
1033         if (status.isSuccess()) {
1034             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1035                          arpProvider == null ? "skipped" : "programmed",
1036                          macAddress, address, dpid, segOrOfPort, action);
1037         } else {
1038             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1039                          macAddress, address, dpid, segOrOfPort, action, status);
1040         }
1041         return status;
1042     }
1043
1044     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1045                                                   String matchAddress, String rewriteAddress,
1046                                                   Action action) {
1047         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1048         // will look at desired action for node.
1049         //
1050         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1051         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1052
1053         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1054             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1055                     " action {} is already done",
1056                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1057             return true;
1058         }
1059         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1060             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1061                     " action is already done",
1062                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1063             return true;
1064         }
1065
1066         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1067                 rewriteAddress, action);
1068         if (status.isSuccess()) {
1069             // Update cache
1070             if (action == Action.ADD) {
1071                 inboundIpRewriteCache.add(cacheKey);
1072             } else {
1073                 inboundIpRewriteCache.remove(cacheKey);
1074             }
1075             return true;
1076         }
1077         return false;
1078     }
1079
1080     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1081                                                  String matchAddress, String rewriteAddress,
1082                                                  Action action) {
1083         Status status;
1084         try {
1085             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1086             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1087             status = inboundNatProvider == null ?
1088                     new Status(StatusCode.SUCCESS) :
1089                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1090                             inetMatchAddress, inetRewriteAddress,
1091                             action);
1092         } catch (UnknownHostException e) {
1093             status = new Status(StatusCode.BADREQUEST);
1094         }
1095
1096         if (status.isSuccess()) {
1097             final boolean isSkipped = inboundNatProvider == null;
1098             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1099                     isSkipped ? "skipped" : "programmed",
1100                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1101         } else {
1102             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1103                          " status:{}",
1104                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1105                     status);
1106         }
1107         return status;
1108     }
1109
1110     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1111                                                  Action actionForRewriteExclusion) {
1112         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1113         // will look at desired action for node.
1114         //
1115         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1116         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1117
1118         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1119             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1120                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1121             return;
1122         }
1123         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1124             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1125                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1126             return;
1127         }
1128
1129         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1130                                                              actionForRewriteExclusion);
1131         if (status.isSuccess()) {
1132             // Update cache
1133             if (actionForRewriteExclusion == Action.ADD) {
1134                     outboundIpRewriteExclusionCache.add(cacheKey);
1135             } else {
1136                     outboundIpRewriteExclusionCache.remove(cacheKey);
1137             }
1138         }
1139     }
1140
1141     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1142                                                    Action actionForNode) {
1143         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1144                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1145
1146         if (status.isSuccess()) {
1147             final boolean isSkipped = outboundNatProvider == null;
1148             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1149                          isSkipped ? "skipped" : "programmed",
1150                          cidr, node.getNodeId().getValue(), actionForNode);
1151         } else {
1152             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1153                          cidr, node.getNodeId().getValue(), actionForNode, status);
1154         }
1155         return status;
1156     }
1157
1158     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1159         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1160         // will look at desired action for node.
1161         //
1162         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1163         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1164
1165         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1166             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1167                          "is already done",
1168                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1169             return;
1170         }
1171         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1172             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1173                          "is already done",
1174                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1175             return;
1176         }
1177
1178         Status status = this.programOutboundIpRewriteStage2(fid, action);
1179         if (status.isSuccess()) {
1180             // Update cache
1181             if (action == Action.ADD) {
1182                 outboundIpRewriteCache.add(cacheKey);
1183             } else {
1184                 outboundIpRewriteCache.remove(cacheKey);
1185             }
1186         }
1187     }
1188
1189     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1190         Status status;
1191         try {
1192             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1193             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1194             status = outboundNatProvider == null ?
1195                     new Status(StatusCode.SUCCESS) :
1196                     outboundNatProvider.programIpRewriteRule(
1197                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1198                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1199         } catch (UnknownHostException e) {
1200             status = new Status(StatusCode.BADREQUEST);
1201         }
1202
1203         if (status.isSuccess()) {
1204             final boolean isSkipped = outboundNatProvider == null;
1205             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1206                             " action {}",
1207                          isSkipped ? "skipped" : "programmed",
1208                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1209         } else {
1210             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1211                          " action {} status:{}",
1212                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1213         }
1214         return status;
1215     }
1216
1217     private int getMaskLenFromCidr(String cidr) {
1218         if (cidr == null) {
1219             return 0;
1220         }
1221         String[] splits = cidr.split("/");
1222         if (splits.length != 2) {
1223             return 0;
1224         }
1225
1226         int result;
1227         try {
1228             result = Integer.parseInt(splits[1].trim());
1229         } catch (NumberFormatException nfe) {
1230             result = 0;
1231         }
1232         return result;
1233     }
1234
1235     private Long getDpidForIntegrationBridge(Node node) {
1236         // Check if node is integration bridge; and only then return its dpid
1237         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1238             return southbound.getDataPathId(node);
1239         }
1240         return null;
1241     }
1242
1243     private Long getDpidForExternalBridge(Node node) {
1244         // Check if node is integration bridge; and only then return its dpid
1245         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1246             return southbound.getDataPathId(node);
1247         }
1248         return null;
1249     }
1250
1251     private Node getExternalBridgeNode(){
1252         //Pickup the first node that has external bridge (br-ex).
1253         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1254         //the external network is reachable from every br-ex
1255         // TODO: Consider other deployment scenario, and thing of better solution.
1256         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1257         for(Node node : allBridges){
1258             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1259                 return node;
1260             }
1261         }
1262         return null;
1263     }
1264
1265     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1266         NeutronSubnet extSubnet = null;
1267         for (NeutronSubnet subnet : neutronSubnetCache.getAllSubnets()){
1268             if(subnet.getPortsInSubnet().contains(gatewayPort)){
1269                 extSubnet = subnet;
1270                 break;
1271             }
1272         }
1273         return extSubnet;
1274     }
1275
1276     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1277
1278         Preconditions.checkNotNull(node);
1279         Preconditions.checkNotNull(gatewayPort);
1280         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1281
1282         if(externalNetwork != null){
1283             if(externalNetwork.isRouterExternal()){
1284                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1285                 if(externalSubnet != null){
1286                     if(externalSubnet.getGatewayIP() != null){
1287                         LOG.info("Trigger MAC resolution for gateway ip {} on Node {}", externalSubnet.getGatewayIP(), node.getNodeId());
1288
1289                         ListenableFuture<MacAddress> gatewayMacAddress =
1290                                 gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1291                                         new Ipv4Address(externalSubnet.getGatewayIP()),
1292                                         new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1293                                         new MacAddress(gatewayPort.getMacAddress()),
1294                                         false);
1295                         if(gatewayMacAddress != null){
1296                             Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1297                                 @Override
1298                                 public void onSuccess(MacAddress result) {
1299                                     if(result != null){
1300                                         updateExternalRouterMac(result.getValue());
1301                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(), result.getValue());
1302                                     }else{
1303                                         LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1304                                     }
1305                                 }
1306
1307                                 @Override
1308                                 public void onFailure(Throwable t) {
1309                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1310                                 }
1311                             }, gatewayMacResolverPool);
1312                         }
1313                     }else{
1314                         LOG.warn("No gateway IP address found for external subnet {}", externalSubnet);
1315                     }
1316                 }else{
1317                     LOG.warn("Neutron subnet not found for external network {}", externalNetwork);
1318                 }
1319             }
1320         }else{
1321             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1322         }
1323     }
1324
1325     /**
1326      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1327      *
1328      * @param ofPort the OF port number
1329      * @return the string with encoded OF port (example format "OFPort|999")
1330      */
1331     public static String encodeExcplicitOFPort(Long ofPort) {
1332         return "OFPort|" + ofPort.toString();
1333     }
1334
1335     @Override
1336     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1337         tenantNetworkManager =
1338                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1339         configurationService =
1340                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1341         arpProvider =
1342                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1343         inboundNatProvider =
1344                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1345         outboundNatProvider =
1346                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1347         routingProvider =
1348                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1349         l3ForwardingProvider =
1350                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1351         nodeCacheManager =
1352                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1353         southbound =
1354                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1355         gatewayMacResolver =
1356                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1357         initL3AdapterMembers();
1358     }
1359
1360     @Override
1361     public void setDependencies(Object impl) {
1362         if (impl instanceof INeutronNetworkCRUD) {
1363             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1364         } else if (impl instanceof INeutronPortCRUD) {
1365             neutronPortCache = (INeutronPortCRUD)impl;
1366         } else if (impl instanceof INeutronSubnetCRUD) {
1367             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1368         } else if (impl instanceof ArpProvider) {
1369             arpProvider = (ArpProvider)impl;
1370         } else if (impl instanceof InboundNatProvider) {
1371             inboundNatProvider = (InboundNatProvider)impl;
1372         } else if (impl instanceof OutboundNatProvider) {
1373             outboundNatProvider = (OutboundNatProvider)impl;
1374         } else if (impl instanceof RoutingProvider) {
1375             routingProvider = (RoutingProvider)impl;
1376         } else if (impl instanceof L3ForwardingProvider) {
1377             l3ForwardingProvider = (L3ForwardingProvider)impl;
1378         }else if (impl instanceof GatewayMacResolver) {
1379             gatewayMacResolver = (GatewayMacResolver)impl;
1380         }
1381     }
1382 }