openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import com.google.common.util.concurrent.FutureCallback;
  34 import com.google.common.util.concurrent.Futures;
  35 import com.google.common.util.concurrent.ListenableFuture;
  36
  37 import org.osgi.framework.BundleContext;
  38 import org.osgi.framework.ServiceReference;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 import java.net.InetAddress;
  43 import java.net.UnknownHostException;
  44 import java.util.HashMap;
  45 import java.util.HashSet;
  46 import java.util.ArrayList;
  47 import java.util.List;
  48 import java.util.Map;
  49 import java.util.Set;
  50 import java.util.concurrent.ExecutorService;
  51 import java.util.concurrent.Executors;
  52
  53 /**
  54  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  55  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  56  * as well as the multi-tenant router forwarding provider.
  57  */
  58 public class NeutronL3Adapter implements ConfigInterface {
  59     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  60
  61     // The implementation for each of these services is resolved by the OSGi Service Manager
  62     private volatile ConfigurationService configurationService;
  63     private volatile TenantNetworkManager tenantNetworkManager;
  64     private volatile NodeCacheManager nodeCacheManager;
  65     private volatile INeutronNetworkCRUD neutronNetworkCache;
  66     private volatile INeutronSubnetCRUD neutronSubnetCache;
  67     private volatile INeutronPortCRUD neutronPortCache;
  68     private volatile L3ForwardingProvider l3ForwardingProvider;
  69     private volatile InboundNatProvider inboundNatProvider;
  70     private volatile OutboundNatProvider outboundNatProvider;
  71     private volatile ArpProvider arpProvider;
  72     private volatile RoutingProvider routingProvider;
  73     private volatile GatewayMacResolver gatewayMacResolver;
  74
  75     private class FloatIpData {
  76         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  77         private final Long ofPort;        // patch port in br-int used to reach br-ex
  78         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  79         private final String macAddress;  // mac address assigned to neutron port of floating ip
  80         private final String floatingIpAddress;
  81         private final String fixedIpAddress;  // ip address given to tenant vm
  82         private final String neutronRouterMac;
  83
  84         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  85                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  86             this.dpid = dpid;
  87             this.ofPort = ofPort;
  88             this.segId = segId;
  89             this.macAddress = macAddress;
  90             this.floatingIpAddress = floatingIpAddress;
  91             this.fixedIpAddress = fixedIpAddress;
  92             this.neutronRouterMac = neutronRouterMac;
  93         }
  94     }
  95
  96     private Set<String> inboundIpRewriteCache;
  97     private Set<String> outboundIpRewriteCache;
  98     private Set<String> outboundIpRewriteExclusionCache;
  99     private Set<String> routerInterfacesCache;
 100     private Set<String> staticArpEntryCache;
 101     private Set<String> l3ForwardingCache;
 102     private Map<String, String> networkIdToRouterMacCache;
 103     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 104     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 105     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 106     private Map<String, FloatIpData> floatIpDataMapCache;
 107     private String externalRouterMac;
 108     private Boolean enabled = false;
 109     private Boolean flgDistributedARPEnabled = true;
 110     private Southbound southbound;
 111     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 112
 113     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 114     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 115     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 116     private static final String OWNER_FLOATING_IP = "network:floatingip";
 117     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 118
 119     public NeutronL3Adapter() {
 120         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 121     }
 122
 123     private void initL3AdapterMembers() {
 124         Preconditions.checkNotNull(configurationService);
 125
 126         if (configurationService.isL3ForwardingEnabled()) {
 127             this.inboundIpRewriteCache = new HashSet<>();
 128             this.outboundIpRewriteCache = new HashSet<>();
 129             this.outboundIpRewriteExclusionCache = new HashSet<>();
 130             this.routerInterfacesCache = new HashSet<>();
 131             this.staticArpEntryCache = new HashSet<>();
 132             this.l3ForwardingCache = new HashSet<>();
 133             this.networkIdToRouterMacCache = new HashMap<>();
 134             this.networkIdToRouterIpListCache = new HashMap<>();
 135             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 136             this.neutronPortToDpIdCache = new HashMap<>();
 137             this.floatIpDataMapCache = new HashMap<>();
 138
 139             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 140             if (this.externalRouterMac == null) {
 141                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 142             }
 143
 144             this.enabled = true;
 145             LOG.info("OVSDB L3 forwarding is enabled");
 146             if (configurationService.isDistributedArpDisabled()) {
 147                 this.flgDistributedARPEnabled = false;
 148                 LOG.info("Distributed ARP responder is disabled");
 149             } else {
 150                 LOG.debug("Distributed ARP responder is enabled");
 151             }
 152         } else {
 153             LOG.debug("OVSDB L3 forwarding is disabled");
 154         }
 155     }
 156
 157     //
 158     // Callbacks from OVSDB's northbound handlers
 159     //
 160
 161     /**
 162      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 163      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 164      * subscribe so that the mac address associated to the gateway ip address is available.
 165      *
 166      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 167      */
 168     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 169         Preconditions.checkNotNull(externalRouterMacUpdate);
 170
 171         flushExistingIpRewrite();
 172         this.externalRouterMac = externalRouterMacUpdate;
 173         rebuildExistingIpRewrite();
 174     }
 175
 176     /**
 177      * Process the event.
 178      *
 179      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 180      * @param subnet An instance of NeutronSubnet object.
 181      */
 182     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 183         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 184     }
 185
 186     /**
 187      * Process the port event as a router interface event.
 188      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 189      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 190      *
 191      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 192      * @param neutronPort An instance of NeutronPort object.
 193      */
 194     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 195         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 196         if (!this.enabled) {
 197             return;
 198         }
 199
 200         final boolean isDelete = action == Action.DELETE;
 201
 202         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 203             if(!isDelete){
 204                 Node externalBridgeNode = getExternalBridgeNode();
 205                 if(externalBridgeNode != null){
 206                     LOG.info("Port {} is network router gateway interface, "
 207                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 208                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 209                 }else{
 210                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 211                 }
 212             }else{
 213                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 214
 215                 if(externalNetwork != null){
 216                     if(externalNetwork.isRouterExternal()){
 217                         final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 218                         if(externalSubnet != null){
 219                             gatewayMacResolver.stopPeriodicReferesh(new Ipv4Address(externalSubnet.getGatewayIP()));
 220                         }
 221                     }
 222                 }
 223             }
 224         }
 225
 226         // Treat the port event as a router interface event if the port belongs to router. This is a
 227         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 228         //
 229         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 230             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 231
 232             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 233                 NeutronRouter_Interface neutronRouterInterface =
 234                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 235                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 236                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 237
 238                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 239             }
 240         } else {
 241             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 242             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 243             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 244             // there.
 245             //
 246             if (!isDelete) {
 247                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 248                     NeutronRouter_Interface neutronRouterInterface =
 249                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 250                     if (neutronRouterInterface != null) {
 251                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 252                     }
 253                 }
 254             }
 255             this.updateL3ForNeutronPort(neutronPort, isDelete);
 256         }
 257     }
 258
 259     /**
 260      * Process the event.
 261      *
 262      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 263      * @param neutronRouter An instance of NeutronRouter object.
 264      */
 265     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 266         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 267     }
 268
 269     /**
 270      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 271      * delete the ports on the same subnet.
 272      *
 273      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 274      * @param neutronRouter An instance of NeutronRouter object.
 275      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 276      */
 277     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 278                                                   final NeutronRouter_Interface neutronRouterInterface,
 279                                                   Action action) {
 280         LOG.debug("Router interface {} got event {}. Subnet {}",
 281                      neutronRouterInterface.getPortUUID(),
 282                      action,
 283                      neutronRouterInterface.getSubnetUUID());
 284         if (!this.enabled) {
 285             return;
 286         }
 287
 288         final boolean isDelete = action == Action.DELETE;
 289
 290         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 291
 292         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 293         // see if they are affected by l3
 294         //
 295         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 296             boolean currPortShouldBeDeleted = false;
 297             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 298             if (isDelete) {
 299                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 300                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 301                         currPortShouldBeDeleted = true;
 302                         break;
 303                     }
 304                 }
 305             }
 306             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 307         }
 308     }
 309
 310     /**
 311      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 312      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 313      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 314      * way around) on OpenFlow Table 100.
 315      *
 316      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 317      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 318      */
 319     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 320                                              Action actionIn) {
 321         Preconditions.checkNotNull(neutronFloatingIP);
 322
 323         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 324                 neutronFloatingIP.getFixedIPAddress(),
 325                 neutronFloatingIP.getFloatingIPAddress(),
 326                 neutronFloatingIP.getFloatingNetworkUUID());
 327         if (!this.enabled) {
 328             return;
 329         }
 330
 331         Action action;
 332
 333         // Consider action to be delete if getFixedIPAddress is null
 334         //
 335         if (neutronFloatingIP.getFixedIPAddress() == null) {
 336             action = Action.DELETE;
 337         } else {
 338             action = actionIn;
 339         }
 340
 341         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 342
 343         if (action != Action.DELETE) {
 344             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 345
 346             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 347             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 348         } else {
 349             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 350             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 351
 352             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 353         }
 354     }
 355
 356     /**
 357      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 358      * ip, otherwise for newer one.
 359      */
 360     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 361         Preconditions.checkNotNull(neutronFloatingIP);
 362
 363         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 364         if (fid == null) {
 365             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 366                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 367             return;
 368         }
 369         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 370                                       action);
 371     }
 372
 373     /**
 374      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 375      * ip, otherwise for newer one.
 376      */
 377     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 378         Preconditions.checkNotNull(neutronFloatingIP);
 379
 380         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 381         if (fid == null) {
 382             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 383                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 384             return;
 385         }
 386         programOutboundIpRewriteStage1(fid, action);
 387     }
 388
 389     private void flushExistingIpRewrite() {
 390         for (FloatIpData fid : floatIpDataMapCache.values()) {
 391             programOutboundIpRewriteStage1(fid, Action.DELETE);
 392         }
 393     }
 394
 395     private void rebuildExistingIpRewrite() {
 396         for (FloatIpData fid : floatIpDataMapCache.values()) {
 397             programOutboundIpRewriteStage1(fid, Action.ADD);
 398         }
 399     }
 400
 401     /**
 402      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 403      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 404      * requests.
 405      */
 406     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 407         Preconditions.checkNotNull(neutronFloatingIP);
 408         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 409         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 410
 411         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 412             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 413                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 414             return;
 415         }
 416
 417         // find bridge Node where floating ip is configured by looking up cache for its port
 418         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 419         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 420         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 421         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 422         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 423         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 424
 425         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 426         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 427                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 428         final String providerSegmentationId = tenantNeutronNetwork != null ?
 429                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 430         final String neutronRouterMac = tenantNeutronNetwork != null ?
 431                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 432
 433         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 434                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 435                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 436                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 437             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 438                     fixedIpAddress,
 439                     floatingIpAddress,
 440                     neutronPortForFloatIp,
 441                     neutronTenantPortUuid,
 442                     providerSegmentationId,
 443                     floatingIpMac,
 444                     neutronRouterMac);
 445             return;
 446         }
 447
 448         // get ofport for patch port in br-int
 449         final Long dpId = nodeIfPair.getLeft();
 450         final Long ofPort = findOFPortForExtPatch(dpId);
 451         if (ofPort == null) {
 452             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 453                     dpId);
 454             return;
 455         }
 456
 457         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 458         //
 459         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 460                 Action.ADD)) {
 461             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 462                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 463             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 464             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 465                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 466                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 467         }
 468     }
 469
 470     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 471         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 472         if (floatIpData == null) {
 473             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 474             return;
 475         }
 476
 477         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 478                 floatIpData.floatingIpAddress, Action.DELETE)) {
 479             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 480             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 481                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 482         }
 483     }
 484
 485     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 486         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 487             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 488                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 489                 return neutronPort;
 490             }
 491         }
 492         return null;
 493     }
 494
 495     private Long findOFPortForExtPatch(Long dpId) {
 496         final String brInt = configurationService.getIntegrationBridgeName();
 497         final String brExt = configurationService.getExternalBridgeName();
 498         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 499
 500         Preconditions.checkNotNull(dpId);
 501         Preconditions.checkNotNull(portNameInt);
 502
 503         final long dpidPrimitive = dpId;
 504         for (Node node : nodeCacheManager.getBridgeNodes()) {
 505             if (dpidPrimitive == southbound.getDataPathId(node)) {
 506                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 507                         southbound.getTerminationPointOfBridge(node, portNameInt);
 508                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 509             }
 510         }
 511         return null;
 512     }
 513
 514     /**
 515      * Process the event.
 516      *
 517      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 518      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 519      */
 520     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 521         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 522     }
 523
 524     //
 525     // Callbacks from OVSDB's southbound handler
 526     //
 527     /**
 528      * Process the event.
 529      *
 530      * @param bridgeNode An instance of Node object.
 531      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 532      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 533      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 534      * object.
 535      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 536      */
 537     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 538                                      final NeutronNetwork neutronNetwork, Action action) {
 539         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 540                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 541         if (!this.enabled) {
 542             return;
 543         }
 544
 545         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 546         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 547         final Uuid interfaceUuid = intf.getInterfaceUuid();
 548
 549         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 550                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 551
 552         if (neutronPort != null) {
 553             final String neutronPortUuid = neutronPort.getPortUUID();
 554
 555             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 556                     dpId != null && interfaceUuid != null) {
 557                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 558             }
 559
 560             handleNeutronPortEvent(neutronPort, action);
 561         }
 562
 563         if (action == Action.DELETE && interfaceUuid != null) {
 564             handleInterfaceEventDelete(intf, dpId);
 565         }
 566     }
 567
 568     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 569         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 570         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 571                 neutronPortUuid, dpId, interfaceUuid.getValue());
 572     }
 573
 574     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 575         // Remove entry from neutronPortToDpIdCache based on interface uuid
 576         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 577             final String currPortUuid = entry.getKey();
 578             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 579                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 580                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 581                 neutronPortToDpIdCache.remove(currPortUuid);
 582                 break;
 583             }
 584         }
 585     }
 586
 587     //
 588     // Internal helpers
 589     //
 590     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 591
 592         final String networkUUID = neutronPort.getNetworkUUID();
 593         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 594
 595         // If there is no router interface handling the networkUUID, we are done
 596         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 597             return;
 598         }
 599
 600         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 601         // router interface are handled via handleNeutronRouterInterfaceEvent.
 602         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 603             return;
 604         }
 605
 606         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 607         final String providerSegmentationId = neutronNetwork != null ?
 608                                               neutronNetwork.getProviderSegmentationID() : null;
 609         final String tenantMac = neutronPort.getMacAddress();
 610
 611         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 612             tenantMac == null || tenantMac.isEmpty()) {
 613             // done: go no further w/out all the info needed...
 614             return;
 615         }
 616
 617         final Action action = isDelete ? Action.DELETE : Action.ADD;
 618         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 619         if (nodes.isEmpty()) {
 620             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 621         }
 622         for (Node node : nodes) {
 623             final Long dpid = getDpidForIntegrationBridge(node);
 624             if (dpid == null) {
 625                 continue;
 626             }
 627
 628             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 629                 final String tenantIpStr = neutronIP.getIpAddress();
 630                 if (tenantIpStr.isEmpty()) {
 631                     continue;
 632                 }
 633
 634                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 635                 // still needed when routing to subnets non-local to node (bug 2076).
 636                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 637
 638                 // Configure distributed ARP responder
 639                 if (flgDistributedARPEnabled) {
 640                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 641                 }
 642             }
 643         }
 644     }
 645
 646     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 647                                            String macAddress, String ipStr,
 648                                            Action actionForNode) {
 649         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 650         // will look at desired action for node.
 651
 652         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 653         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 654
 655         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 656             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 657                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 658             return;
 659         }
 660         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 661             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 662                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 663             return;
 664         }
 665
 666         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 667                                                        macAddress, ipStr, actionForNode);
 668         if (status.isSuccess()) {
 669             // Update cache
 670             if (actionForNode == Action.ADD) {
 671                 l3ForwardingCache.add(cacheKey);
 672             } else {
 673                 l3ForwardingCache.remove(cacheKey);
 674             }
 675         }
 676     }
 677
 678     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 679                                              String macAddress,
 680                                              String address,
 681                                              Action actionForNode) {
 682         Status status;
 683         try {
 684             InetAddress inetAddress = InetAddress.getByName(address);
 685             status = l3ForwardingProvider == null ?
 686                      new Status(StatusCode.SUCCESS) :
 687                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 688                                                                       inetAddress, macAddress, actionForNode);
 689         } catch (UnknownHostException e) {
 690             status = new Status(StatusCode.BADREQUEST);
 691         }
 692
 693         if (status.isSuccess()) {
 694             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 695                          l3ForwardingProvider == null ? "skipped" : "programmed",
 696                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 697         } else {
 698             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 699                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 700         }
 701         return status;
 702     }
 703
 704     // --
 705
 706     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 707                                                        Boolean isDelete) {
 708         Preconditions.checkNotNull(destNeutronRouterInterface);
 709
 710         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 711         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 712         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 713         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 714         final NeutronNetwork neutronNetwork = subnet != null ?
 715                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 716         final String destinationSegmentationId = neutronNetwork != null ?
 717                                                  neutronNetwork.getProviderSegmentationID() : null;
 718         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 719         final String cidr = subnet != null ? subnet.getCidr() : null;
 720         final int mask = getMaskLenFromCidr(cidr);
 721
 722         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 723                      destNeutronRouterInterface, isDelete);
 724
 725         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 726         // the local cache
 727         if (neutronNetwork != null) {
 728             if (macAddress == null || macAddress.isEmpty()) {
 729                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 730             }
 731             if (ipList == null || ipList.isEmpty()) {
 732                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 733             }
 734         }
 735
 736         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 737             cidr == null || cidr.isEmpty() ||
 738             macAddress == null || macAddress.isEmpty() ||
 739             ipList == null || ipList.isEmpty()) {
 740             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 741                          destinationSegmentationId, cidr, macAddress, ipList);
 742             // done: go no further w/out all the info needed...
 743             return;
 744         }
 745
 746         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 747
 748         // Keep cache for finding router's mac from network uuid -- add
 749         //
 750         if (! isDelete) {
 751             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 752             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 753             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 754         }
 755
 756         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 757         if (nodes.isEmpty()) {
 758             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 759         }
 760         for (Node node : nodes) {
 761             final Long dpid = getDpidForIntegrationBridge(node);
 762             if (dpid == null) {
 763                 continue;
 764             }
 765
 766             for (Neutron_IPs neutronIP : ipList) {
 767                 final String ipStr = neutronIP.getIpAddress();
 768                 if (ipStr.isEmpty()) {
 769                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 770                             node.getNodeId().getValue(), ipStr);
 771                     continue;
 772                 }
 773
 774                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 775                 //
 776                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 777                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 778                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 779                                                               neutronNetwork, destinationSegmentationId,
 780                                                               macAddress, ipStr, mask, actionForNode,
 781                                                               true /*isReflexsive*/);
 782                 }
 783
 784                 if (! isExternal) {
 785                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 786                             actionForNode);
 787                 }
 788                 // Enable ARP responder by default, because router interface needs to be responded always.
 789                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 790             }
 791
 792             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 793             // for the external neutron networks.
 794             //
 795             {
 796                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 797                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 798             }
 799         }
 800
 801         // Keep cache for finding router's mac from network uuid -- remove
 802         //
 803         if (isDelete) {
 804             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 805             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 806             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 807         }
 808     }
 809
 810     private void programFlowForNetworkFromExternal(final Node node,
 811                                                    final Long dpid,
 812                                                    final String destinationSegmentationId,
 813                                                    final String dstMacAddress,
 814                                                    final String destIpStr,
 815                                                    final int destMask,
 816                                                    final Action actionForNode) {
 817         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 818                 dstMacAddress, destIpStr, destMask, actionForNode);
 819     }
 820
 821     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 822                                                            final Long dpid,
 823                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 824                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 825                                                            final NeutronNetwork dstNeutronNetwork,
 826                                                            final String destinationSegmentationId,
 827                                                            final String dstMacAddress,
 828                                                            final String destIpStr,
 829                                                            final int destMask,
 830                                                            final Action actionForNode,
 831                                                            Boolean isReflexsive) {
 832         Preconditions.checkNotNull(srcNeutronRouterInterface);
 833         Preconditions.checkNotNull(dstNeutronRouterInterface);
 834
 835         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 836         if (sourceSubnetId == null) {
 837             LOG.error("Could not get provider Subnet ID from router interface {}",
 838                          srcNeutronRouterInterface.getID());
 839             return;
 840         }
 841
 842         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 843         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 844         if (sourceNetworkId == null) {
 845             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 846             return;
 847         }
 848
 849         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 850         if (sourceNetwork == null) {
 851             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 852             return;
 853         }
 854
 855         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 856             // Isolate subnets from different tenants within the same router
 857             return;
 858         }
 859         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 860         if (sourceSegmentationId == null) {
 861             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 862             return;
 863         }
 864         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 865             // Skip 'self'
 866             return;
 867         }
 868
 869         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 870                                      dstMacAddress, destIpStr, destMask, actionForNode);
 871
 872         // Flip roles src->dst; dst->src
 873         if (isReflexsive) {
 874             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 875             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 876             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 877             final String cidr2 = sourceSubnet.getCidr();
 878             final int mask2 = getMaskLenFromCidr(cidr2);
 879
 880             if (cidr2 == null || cidr2.isEmpty() ||
 881                 macAddress2 == null || macAddress2.isEmpty() ||
 882                 ipList2 == null || ipList2.isEmpty()) {
 883                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 884                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 885                 // done: go no further w/out all the info needed...
 886                 return;
 887             }
 888
 889             for (Neutron_IPs neutronIP2 : ipList2) {
 890                 final String ipStr2 = neutronIP2.getIpAddress();
 891                 if (ipStr2.isEmpty()) {
 892                     continue;
 893                 }
 894                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 895                                                           srcNeutronRouterInterface,
 896                                                           sourceNetwork, sourceSegmentationId,
 897                                                           macAddress2, ipStr2, mask2, actionForNode,
 898                                                           false /*isReflexsive*/);
 899             }
 900         }
 901     }
 902
 903     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 904                                               String destinationSegmentationId,
 905                                               String macAddress, String ipStr, int mask,
 906                                               Action actionForNode) {
 907         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 908         // will look at desired action for node.
 909         //
 910         final String cacheKey = node.getNodeId().getValue() + ":" +
 911                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 912                                 ipStr + "/" + Integer.toString(mask);
 913         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 914
 915         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 916             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 917                          " action {} is already done",
 918                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 919                          macAddress, ipStr, mask, actionForNode);
 920             return;
 921         }
 922         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 923             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 924                          " action {} is already done",
 925                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 926                          macAddress, ipStr, mask, actionForNode);
 927             return;
 928         }
 929
 930         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 931                                                           macAddress, ipStr, mask, actionForNode);
 932         if (status.isSuccess()) {
 933             // Update cache
 934             if (actionForNode == Action.ADD) {
 935                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 936                 routerInterfacesCache.add(cacheKey);
 937             } else {
 938                 // TODO: multiTenantAwareRouter.removeInterface(...);
 939                 routerInterfacesCache.remove(cacheKey);
 940             }
 941         }
 942     }
 943
 944     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 945                                                 String destinationSegmentationId,
 946                                                 String macAddress,
 947                                                 String address, int mask,
 948                                                 Action actionForNode) {
 949         Status status;
 950         try {
 951             InetAddress inetAddress = InetAddress.getByName(address);
 952             status = routingProvider == null ?
 953                      new Status(StatusCode.SUCCESS) :
 954                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 955                                                             macAddress, inetAddress, mask, actionForNode);
 956         } catch (UnknownHostException e) {
 957             status = new Status(StatusCode.BADREQUEST);
 958         }
 959
 960         if (status.isSuccess()) {
 961             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 962                          routingProvider == null ? "skipped" : "programmed",
 963                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 964                          actionForNode);
 965         } else {
 966             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 967                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 968                          actionForNode, status);
 969         }
 970         return status;
 971     }
 972
 973     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 974                                            String macAddress, String ipStr,
 975                                            Action action) {
 976         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 977         // will look at desired action for node.
 978         //
 979         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 980         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 981
 982         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 983             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 984                     dpid, segOrOfPort, macAddress, ipStr, action);
 985             return true;
 986         }
 987         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 988             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 989                     dpid, segOrOfPort, macAddress, ipStr, action);
 990             return true;
 991         }
 992
 993         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 994         if (status.isSuccess()) {
 995             // Update cache
 996             if (action == Action.ADD) {
 997                 staticArpEntryCache.add(cacheKey);
 998             } else {
 999                 staticArpEntryCache.remove(cacheKey);
1000             }
1001             return true;
1002         }
1003         return false;
1004     }
1005
1006     private Status programStaticArpStage2(Long dpid,
1007                                           String segOrOfPort,
1008                                           String macAddress,
1009                                           String address,
1010                                           Action action) {
1011         Status status;
1012         try {
1013             InetAddress inetAddress = InetAddress.getByName(address);
1014             status = arpProvider == null ?
1015                      new Status(StatusCode.SUCCESS) :
1016                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1017                                                        macAddress, inetAddress, action);
1018         } catch (UnknownHostException e) {
1019             status = new Status(StatusCode.BADREQUEST);
1020         }
1021
1022         if (status.isSuccess()) {
1023             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1024                          arpProvider == null ? "skipped" : "programmed",
1025                          macAddress, address, dpid, segOrOfPort, action);
1026         } else {
1027             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1028                          macAddress, address, dpid, segOrOfPort, action, status);
1029         }
1030         return status;
1031     }
1032
1033     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1034                                                   String matchAddress, String rewriteAddress,
1035                                                   Action action) {
1036         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1037         // will look at desired action for node.
1038         //
1039         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1040         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1041
1042         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1043             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1044                     " action {} is already done",
1045                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1046             return true;
1047         }
1048         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1049             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1050                     " action is already done",
1051                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1052             return true;
1053         }
1054
1055         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1056                 rewriteAddress, action);
1057         if (status.isSuccess()) {
1058             // Update cache
1059             if (action == Action.ADD) {
1060                 inboundIpRewriteCache.add(cacheKey);
1061             } else {
1062                 inboundIpRewriteCache.remove(cacheKey);
1063             }
1064             return true;
1065         }
1066         return false;
1067     }
1068
1069     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1070                                                  String matchAddress, String rewriteAddress,
1071                                                  Action action) {
1072         Status status;
1073         try {
1074             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1075             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1076             status = inboundNatProvider == null ?
1077                     new Status(StatusCode.SUCCESS) :
1078                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1079                             inetMatchAddress, inetRewriteAddress,
1080                             action);
1081         } catch (UnknownHostException e) {
1082             status = new Status(StatusCode.BADREQUEST);
1083         }
1084
1085         if (status.isSuccess()) {
1086             final boolean isSkipped = inboundNatProvider == null;
1087             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1088                     isSkipped ? "skipped" : "programmed",
1089                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1090         } else {
1091             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1092                          " status:{}",
1093                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1094                     status);
1095         }
1096         return status;
1097     }
1098
1099     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1100                                                  Action actionForRewriteExclusion) {
1101         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1102         // will look at desired action for node.
1103         //
1104         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1105         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1106
1107         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1108             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1109                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1110             return;
1111         }
1112         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1113             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1114                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1115             return;
1116         }
1117
1118         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1119                                                              actionForRewriteExclusion);
1120         if (status.isSuccess()) {
1121             // Update cache
1122             if (actionForRewriteExclusion == Action.ADD) {
1123                     outboundIpRewriteExclusionCache.add(cacheKey);
1124             } else {
1125                     outboundIpRewriteExclusionCache.remove(cacheKey);
1126             }
1127         }
1128     }
1129
1130     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1131                                                    Action actionForNode) {
1132         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1133                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1134
1135         if (status.isSuccess()) {
1136             final boolean isSkipped = outboundNatProvider == null;
1137             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1138                          isSkipped ? "skipped" : "programmed",
1139                          cidr, node.getNodeId().getValue(), actionForNode);
1140         } else {
1141             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1142                          cidr, node.getNodeId().getValue(), actionForNode, status);
1143         }
1144         return status;
1145     }
1146
1147     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1148         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1149         // will look at desired action for node.
1150         //
1151         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1152         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1153
1154         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1155             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1156                          "is already done",
1157                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1158             return;
1159         }
1160         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1161             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1162                          "is already done",
1163                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1164             return;
1165         }
1166
1167         Status status = this.programOutboundIpRewriteStage2(fid, action);
1168         if (status.isSuccess()) {
1169             // Update cache
1170             if (action == Action.ADD) {
1171                 outboundIpRewriteCache.add(cacheKey);
1172             } else {
1173                 outboundIpRewriteCache.remove(cacheKey);
1174             }
1175         }
1176     }
1177
1178     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1179         Status status;
1180         try {
1181             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1182             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1183             status = outboundNatProvider == null ?
1184                     new Status(StatusCode.SUCCESS) :
1185                     outboundNatProvider.programIpRewriteRule(
1186                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1187                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1188         } catch (UnknownHostException e) {
1189             status = new Status(StatusCode.BADREQUEST);
1190         }
1191
1192         if (status.isSuccess()) {
1193             final boolean isSkipped = outboundNatProvider == null;
1194             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1195                             " action {}",
1196                          isSkipped ? "skipped" : "programmed",
1197                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1198         } else {
1199             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1200                          " action {} status:{}",
1201                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1202         }
1203         return status;
1204     }
1205
1206     private int getMaskLenFromCidr(String cidr) {
1207         if (cidr == null) {
1208             return 0;
1209         }
1210         String[] splits = cidr.split("/");
1211         if (splits.length != 2) {
1212             return 0;
1213         }
1214
1215         int result;
1216         try {
1217             result = Integer.parseInt(splits[1].trim());
1218         } catch (NumberFormatException nfe) {
1219             result = 0;
1220         }
1221         return result;
1222     }
1223
1224     private Long getDpidForIntegrationBridge(Node node) {
1225         // Check if node is integration bridge; and only then return its dpid
1226         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1227             return southbound.getDataPathId(node);
1228         }
1229         return null;
1230     }
1231
1232     private Long getDpidForExternalBridge(Node node) {
1233         // Check if node is integration bridge; and only then return its dpid
1234         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1235             return southbound.getDataPathId(node);
1236         }
1237         return null;
1238     }
1239
1240     private Node getExternalBridgeNode(){
1241         //Pickup the first node that has external bridge (br-ex).
1242         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1243         //the external network is reachable from every br-ex
1244         // TODO: Consider other deployment scenario, and thing of better solution.
1245         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1246         for(Node node : allBridges){
1247             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1248                 return node;
1249             }
1250         }
1251         return null;
1252     }
1253
1254     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1255         NeutronSubnet extSubnet = null;
1256         for (NeutronSubnet subnet : neutronSubnetCache.getAllSubnets()){
1257             if(subnet.getPortsInSubnet().contains(gatewayPort)){
1258                 extSubnet = subnet;
1259                 break;
1260             }
1261         }
1262         return extSubnet;
1263     }
1264
1265     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1266
1267         Preconditions.checkNotNull(node);
1268         Preconditions.checkNotNull(gatewayPort);
1269         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1270
1271         if(externalNetwork != null){
1272             if(externalNetwork.isRouterExternal()){
1273                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1274                 if(externalSubnet != null){
1275                     if(externalSubnet.getGatewayIP() != null){
1276                         LOG.info("Trigger MAC resolution for gateway ip {} on Node {}", externalSubnet.getGatewayIP(), node.getNodeId());
1277
1278                         ListenableFuture<MacAddress> gatewayMacAddress =
1279                                 gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1280                                         new Ipv4Address(externalSubnet.getGatewayIP()),
1281                                         new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1282                                         new MacAddress(gatewayPort.getMacAddress()),
1283                                         false);
1284                         if(gatewayMacAddress != null){
1285                             Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1286                                 @Override
1287                                 public void onSuccess(MacAddress result) {
1288                                     if(result != null){
1289                                         updateExternalRouterMac(result.getValue());
1290                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(), result.getValue());
1291                                     }else{
1292                                         LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1293                                     }
1294                                 }
1295
1296                                 @Override
1297                                 public void onFailure(Throwable t) {
1298                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1299                                 }
1300                             }, gatewayMacResolverPool);
1301                         }
1302                     }else{
1303                         LOG.warn("No gateway IP address found for external subnet {}", externalSubnet);
1304                     }
1305                 }else{
1306                     LOG.warn("Neutron subnet not found for external network {}", externalNetwork);
1307                 }
1308             }
1309         }else{
1310             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1311         }
1312     }
1313
1314     /**
1315      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1316      *
1317      * @param ofPort the OF port number
1318      * @return the string with encoded OF port (example format "OFPort|999")
1319      */
1320     public static String encodeExcplicitOFPort(Long ofPort) {
1321         return "OFPort|" + ofPort.toString();
1322     }
1323
1324     @Override
1325     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1326         tenantNetworkManager =
1327                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1328         configurationService =
1329                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1330         arpProvider =
1331                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1332         inboundNatProvider =
1333                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1334         outboundNatProvider =
1335                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1336         routingProvider =
1337                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1338         l3ForwardingProvider =
1339                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1340         nodeCacheManager =
1341                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1342         southbound =
1343                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1344         gatewayMacResolver =
1345                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1346         initL3AdapterMembers();
1347     }
1348
1349     @Override
1350     public void setDependencies(Object impl) {
1351         if (impl instanceof INeutronNetworkCRUD) {
1352             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1353         } else if (impl instanceof INeutronPortCRUD) {
1354             neutronPortCache = (INeutronPortCRUD)impl;
1355         } else if (impl instanceof INeutronSubnetCRUD) {
1356             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1357         } else if (impl instanceof ArpProvider) {
1358             arpProvider = (ArpProvider)impl;
1359         } else if (impl instanceof InboundNatProvider) {
1360             inboundNatProvider = (InboundNatProvider)impl;
1361         } else if (impl instanceof OutboundNatProvider) {
1362             outboundNatProvider = (OutboundNatProvider)impl;
1363         } else if (impl instanceof RoutingProvider) {
1364             routingProvider = (RoutingProvider)impl;
1365         } else if (impl instanceof L3ForwardingProvider) {
1366             l3ForwardingProvider = (L3ForwardingProvider)impl;
1367         }else if (impl instanceof GatewayMacResolver) {
1368             gatewayMacResolver = (GatewayMacResolver)impl;
1369         }
1370     }
1371 }