openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  14 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  15 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  16 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  17 import org.opendaylight.neutron.spi.NeutronNetwork;
  18 import org.opendaylight.neutron.spi.NeutronPort;
  19 import org.opendaylight.neutron.spi.NeutronRouter;
  20 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  21 import org.opendaylight.neutron.spi.NeutronSubnet;
  22 import org.opendaylight.neutron.spi.Neutron_IPs;
  23 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  25 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  26 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  29 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  30 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  31
  32 import com.google.common.base.Preconditions;
  33 import com.google.common.util.concurrent.FutureCallback;
  34 import com.google.common.util.concurrent.Futures;
  35 import com.google.common.util.concurrent.ListenableFuture;
  36
  37 import org.osgi.framework.ServiceReference;
  38 import org.slf4j.Logger;
  39 import org.slf4j.LoggerFactory;
  40
  41 import java.net.InetAddress;
  42 import java.net.UnknownHostException;
  43 import java.util.HashMap;
  44 import java.util.HashSet;
  45 import java.util.ArrayList;
  46 import java.util.List;
  47 import java.util.Map;
  48 import java.util.Set;
  49 import java.util.concurrent.ExecutorService;
  50 import java.util.concurrent.Executors;
  51
  52 /**
  53  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  54  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  55  * as well as the multi-tenant router forwarding provider.
  56  */
  57 public class NeutronL3Adapter implements ConfigInterface {
  58     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  59
  60     // The implementation for each of these services is resolved by the OSGi Service Manager
  61     private volatile ConfigurationService configurationService;
  62     private volatile TenantNetworkManager tenantNetworkManager;
  63     private volatile NodeCacheManager nodeCacheManager;
  64     private volatile INeutronNetworkCRUD neutronNetworkCache;
  65     private volatile INeutronSubnetCRUD neutronSubnetCache;
  66     private volatile INeutronPortCRUD neutronPortCache;
  67     private volatile L3ForwardingProvider l3ForwardingProvider;
  68     private volatile InboundNatProvider inboundNatProvider;
  69     private volatile OutboundNatProvider outboundNatProvider;
  70     private volatile ArpProvider arpProvider;
  71     private volatile RoutingProvider routingProvider;
  72     private volatile GatewayMacResolver gatewayMacResolver;
  73
  74     private class FloatIpData {
  75         // br-int of node where floating ip is associated with tenant port
  76         private final Long dpid;
  77         // patch port in br-int used to reach br-ex
  78         private final Long ofPort;
  79         // segmentation id of the net where fixed ip is instantiated
  80         private final String segId;
  81         // mac address assigned to neutron port of floating ip
  82         private final String macAddress;
  83         private final String floatingIpAddress;
  84         // ip address given to tenant vm
  85         private final String fixedIpAddress;
  86         private final String neutronRouterMac;
  87
  88         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  89                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  90             this.dpid = dpid;
  91             this.ofPort = ofPort;
  92             this.segId = segId;
  93             this.macAddress = macAddress;
  94             this.floatingIpAddress = floatingIpAddress;
  95             this.fixedIpAddress = fixedIpAddress;
  96             this.neutronRouterMac = neutronRouterMac;
  97         }
  98     }
  99
 100     private Set<String> inboundIpRewriteCache;
 101     private Set<String> outboundIpRewriteCache;
 102     private Set<String> outboundIpRewriteExclusionCache;
 103     private Set<String> routerInterfacesCache;
 104     private Set<String> staticArpEntryCache;
 105     private Set<String> l3ForwardingCache;
 106     private Map<String, String> networkIdToRouterMacCache;
 107     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 108     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 109     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 110     private Map<String, FloatIpData> floatIpDataMapCache;
 111     private String externalRouterMac;
 112     private Boolean enabled = false;
 113     private Boolean flgDistributedARPEnabled = true;
 114     private Southbound southbound;
 115     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 116
 117     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 118     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 119     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 120     private static final String OWNER_FLOATING_IP = "network:floatingip";
 121     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 122
 123     public NeutronL3Adapter() {
 124         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 125     }
 126
 127     private void initL3AdapterMembers() {
 128         Preconditions.checkNotNull(configurationService);
 129
 130         if (configurationService.isL3ForwardingEnabled()) {
 131             this.inboundIpRewriteCache = new HashSet<>();
 132             this.outboundIpRewriteCache = new HashSet<>();
 133             this.outboundIpRewriteExclusionCache = new HashSet<>();
 134             this.routerInterfacesCache = new HashSet<>();
 135             this.staticArpEntryCache = new HashSet<>();
 136             this.l3ForwardingCache = new HashSet<>();
 137             this.networkIdToRouterMacCache = new HashMap<>();
 138             this.networkIdToRouterIpListCache = new HashMap<>();
 139             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 140             this.neutronPortToDpIdCache = new HashMap<>();
 141             this.floatIpDataMapCache = new HashMap<>();
 142
 143             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 144             if (this.externalRouterMac == null) {
 145                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 146             }
 147
 148             this.enabled = true;
 149             LOG.info("OVSDB L3 forwarding is enabled");
 150             if (configurationService.isDistributedArpDisabled()) {
 151                 this.flgDistributedARPEnabled = false;
 152                 LOG.info("Distributed ARP responder is disabled");
 153             } else {
 154                 LOG.debug("Distributed ARP responder is enabled");
 155             }
 156         } else {
 157             LOG.debug("OVSDB L3 forwarding is disabled");
 158         }
 159     }
 160
 161     //
 162     // Callbacks from OVSDB's northbound handlers
 163     //
 164
 165     /**
 166      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 167      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 168      * subscribe so that the mac address associated to the gateway ip address is available.
 169      *
 170      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 171      */
 172     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 173         Preconditions.checkNotNull(externalRouterMacUpdate);
 174
 175         flushExistingIpRewrite();
 176         this.externalRouterMac = externalRouterMacUpdate;
 177         rebuildExistingIpRewrite();
 178     }
 179
 180     /**
 181      * Process the event.
 182      *
 183      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 184      * @param subnet An instance of NeutronSubnet object.
 185      */
 186     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 187         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 188     }
 189
 190     /**
 191      * Process the port event as a router interface event.
 192      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 193      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 194      *
 195      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 196      * @param neutronPort An instance of NeutronPort object.
 197      */
 198     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 199         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 200         if (!this.enabled) {
 201             return;
 202         }
 203
 204         final boolean isDelete = action == Action.DELETE;
 205
 206         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 207             if(!isDelete){
 208                 Node externalBridgeNode = getExternalBridgeNode();
 209                 if(externalBridgeNode != null){
 210                     LOG.info("Port {} is network router gateway interface, "
 211                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 212                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 213                 }else{
 214                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 215                 }
 216             }else{
 217                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 218
 219                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 220                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 221                     // TODO support IPv6
 222                     if (externalSubnet != null &&
 223                             externalSubnet.getIpVersion() == 4) {
 224                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 225                     }
 226                 }
 227             }
 228         }
 229
 230         // Treat the port event as a router interface event if the port belongs to router. This is a
 231         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 232         //
 233         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 234             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 235
 236             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 237                 NeutronRouter_Interface neutronRouterInterface =
 238                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 239                 // id of router interface to be same as subnet
 240                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 241                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 242
 243                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 244             }
 245         } else {
 246             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 247             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 248             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 249             // there.
 250             //
 251             if (!isDelete && neutronPort.getFixedIPs() != null) {
 252                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 253                     NeutronRouter_Interface neutronRouterInterface =
 254                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 255                     if (neutronRouterInterface != null) {
 256                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 257                     }
 258                 }
 259             }
 260             this.updateL3ForNeutronPort(neutronPort, isDelete);
 261         }
 262     }
 263
 264     /**
 265      * Process the event.
 266      *
 267      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 268      * @param neutronRouter An instance of NeutronRouter object.
 269      */
 270     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 271         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 272     }
 273
 274     /**
 275      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 276      * delete the ports on the same subnet.
 277      *
 278      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 279      * @param neutronRouter An instance of NeutronRouter object.
 280      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 281      */
 282     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 283                                                   final NeutronRouter_Interface neutronRouterInterface,
 284                                                   Action action) {
 285         LOG.debug("Router interface {} got event {}. Subnet {}",
 286                      neutronRouterInterface.getPortUUID(),
 287                      action,
 288                      neutronRouterInterface.getSubnetUUID());
 289         if (!this.enabled) {
 290             return;
 291         }
 292
 293         final boolean isDelete = action == Action.DELETE;
 294
 295         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 296
 297         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 298         // see if they are affected by l3
 299         //
 300         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 301             boolean currPortShouldBeDeleted = false;
 302             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 303             if (isDelete) {
 304                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 305                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 306                         currPortShouldBeDeleted = true;
 307                         break;
 308                     }
 309                 }
 310             }
 311             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 312         }
 313
 314         if (isDelete) {
 315             /*
 316              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 317              */
 318             this.cleanupRouterCache(neutronRouterInterface);
 319         }
 320     }
 321
 322     /**
 323      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 324      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 325      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 326      * way around) on OpenFlow Table 100.
 327      *
 328      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 329      * @param neutronFloatingIP An {@link org.opendaylight.neutron.spi.NeutronFloatingIP} instance of NeutronFloatingIP object.
 330      */
 331     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 332                                              Action actionIn) {
 333         Preconditions.checkNotNull(neutronFloatingIP);
 334
 335         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 336                 neutronFloatingIP.getFixedIPAddress(),
 337                 neutronFloatingIP.getFloatingIPAddress(),
 338                 neutronFloatingIP.getFloatingNetworkUUID());
 339         if (!this.enabled) {
 340             return;
 341         }
 342
 343         Action action;
 344
 345         // Consider action to be delete if getFixedIPAddress is null
 346         //
 347         if (neutronFloatingIP.getFixedIPAddress() == null) {
 348             action = Action.DELETE;
 349         } else {
 350             action = actionIn;
 351         }
 352
 353         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 354
 355         if (action != Action.DELETE) {
 356             // must be first, as it updates floatIpDataMapCache
 357             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 358
 359             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 360             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 361         } else {
 362             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 363             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 364
 365             // must be last, as it updates floatIpDataMapCache
 366             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 367         }
 368     }
 369
 370     /**
 371      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 372      * ip, otherwise for newer one.
 373      */
 374     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 375         Preconditions.checkNotNull(neutronFloatingIP);
 376
 377         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 378         if (fid == null) {
 379             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 380                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 381             return;
 382         }
 383         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 384                                       action);
 385     }
 386
 387     /**
 388      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 389      * ip, otherwise for newer one.
 390      */
 391     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 392         Preconditions.checkNotNull(neutronFloatingIP);
 393
 394         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 395         if (fid == null) {
 396             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 397                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 398             return;
 399         }
 400         programOutboundIpRewriteStage1(fid, action);
 401     }
 402
 403     private void flushExistingIpRewrite() {
 404         for (FloatIpData fid : floatIpDataMapCache.values()) {
 405             programOutboundIpRewriteStage1(fid, Action.DELETE);
 406         }
 407     }
 408
 409     private void rebuildExistingIpRewrite() {
 410         for (FloatIpData fid : floatIpDataMapCache.values()) {
 411             programOutboundIpRewriteStage1(fid, Action.ADD);
 412         }
 413     }
 414
 415     /**
 416      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 417      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 418      * requests.
 419      */
 420     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 421         Preconditions.checkNotNull(neutronFloatingIP);
 422         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 423         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 424
 425         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 426             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 427                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 428             return;
 429         }
 430
 431         // find bridge Node where floating ip is configured by looking up cache for its port
 432         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 433         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 434         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 435         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 436         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 437         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 438
 439         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 440         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 441                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 442         final String providerSegmentationId = tenantNeutronNetwork != null ?
 443                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 444         final String neutronRouterMac = tenantNeutronNetwork != null ?
 445                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 446
 447         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 448                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 449                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 450                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 451             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 452                     fixedIpAddress,
 453                     floatingIpAddress,
 454                     neutronPortForFloatIp,
 455                     neutronTenantPortUuid,
 456                     providerSegmentationId,
 457                     floatingIpMac,
 458                     neutronRouterMac);
 459             return;
 460         }
 461
 462         // get ofport for patch port in br-int
 463         final Long dpId = nodeIfPair.getLeft();
 464         final Long ofPort = findOFPortForExtPatch(dpId);
 465         if (ofPort == null) {
 466             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 467                     dpId);
 468             return;
 469         }
 470
 471         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 472         //
 473         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 474                 Action.ADD)) {
 475             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 476                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 477             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 478             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 479                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 480                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 481         }
 482     }
 483
 484     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 485         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 486         if (floatIpData == null) {
 487             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 488             return;
 489         }
 490
 491         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 492                 floatIpData.floatingIpAddress, Action.DELETE)) {
 493             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 494             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 495                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 496         }
 497     }
 498
 499     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 500         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 501             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 502                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 503                 return neutronPort;
 504             }
 505         }
 506         return null;
 507     }
 508
 509     private Long findOFPortForExtPatch(Long dpId) {
 510         final String brInt = configurationService.getIntegrationBridgeName();
 511         final String brExt = configurationService.getExternalBridgeName();
 512         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 513
 514         Preconditions.checkNotNull(dpId);
 515         Preconditions.checkNotNull(portNameInt);
 516
 517         final long dpidPrimitive = dpId;
 518         for (Node node : nodeCacheManager.getBridgeNodes()) {
 519             if (dpidPrimitive == southbound.getDataPathId(node)) {
 520                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 521                         southbound.getTerminationPointOfBridge(node, portNameInt);
 522                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 523             }
 524         }
 525         return null;
 526     }
 527
 528     /**
 529      * Process the event.
 530      *
 531      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 532      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronFloatingIP object.
 533      */
 534     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 535         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 536     }
 537
 538     //
 539     // Callbacks from OVSDB's southbound handler
 540     //
 541     /**
 542      * Process the event.
 543      *
 544      * @param bridgeNode An instance of Node object.
 545      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 546      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 547      * @param neutronNetwork An {@link org.opendaylight.neutron.spi.NeutronNetwork} instance of NeutronNetwork
 548      * object.
 549      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 550      */
 551     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 552                                      final NeutronNetwork neutronNetwork, Action action) {
 553         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 554                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 555         if (!this.enabled) {
 556             return;
 557         }
 558
 559         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 560         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 561         final Uuid interfaceUuid = intf.getInterfaceUuid();
 562
 563         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 564                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 565
 566         if (neutronPort != null) {
 567             final String neutronPortUuid = neutronPort.getPortUUID();
 568
 569             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 570                     dpId != null && interfaceUuid != null) {
 571                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 572             }
 573
 574             handleNeutronPortEvent(neutronPort, action);
 575         }
 576
 577         if (action == Action.DELETE && interfaceUuid != null) {
 578             handleInterfaceEventDelete(intf, dpId);
 579         }
 580     }
 581
 582     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 583         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 584         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 585                 neutronPortUuid, dpId, interfaceUuid.getValue());
 586     }
 587
 588     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 589         // Remove entry from neutronPortToDpIdCache based on interface uuid
 590         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 591             final String currPortUuid = entry.getKey();
 592             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 593                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 594                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 595                 neutronPortToDpIdCache.remove(currPortUuid);
 596                 break;
 597             }
 598         }
 599     }
 600
 601     //
 602     // Internal helpers
 603     //
 604     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 605
 606         final String networkUUID = neutronPort.getNetworkUUID();
 607         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 608
 609         // If there is no router interface handling the networkUUID, we are done
 610         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 611             return;
 612         }
 613
 614         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 615         // router interface are handled via handleNeutronRouterInterfaceEvent.
 616         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 617             return;
 618         }
 619
 620         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 621         final String providerSegmentationId = neutronNetwork != null ?
 622                                               neutronNetwork.getProviderSegmentationID() : null;
 623         final String tenantMac = neutronPort.getMacAddress();
 624
 625         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 626             tenantMac == null || tenantMac.isEmpty()) {
 627             // done: go no further w/out all the info needed...
 628             return;
 629         }
 630
 631         final Action action = isDelete ? Action.DELETE : Action.ADD;
 632         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 633         if (nodes.isEmpty()) {
 634             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 635         }
 636         for (Node node : nodes) {
 637             final Long dpid = getDpidForIntegrationBridge(node);
 638             if (dpid == null) {
 639                 continue;
 640             }
 641
 642             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 643                 final String tenantIpStr = neutronIP.getIpAddress();
 644                 if (tenantIpStr.isEmpty()) {
 645                     continue;
 646                 }
 647
 648                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 649                 // still needed when routing to subnets non-local to node (bug 2076).
 650                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 651
 652                 // Configure distributed ARP responder
 653                 if (flgDistributedARPEnabled) {
 654                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 655                 }
 656             }
 657         }
 658     }
 659
 660     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 661                                            String macAddress, String ipStr,
 662                                            Action actionForNode) {
 663         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 664         // will look at desired action for node.
 665
 666         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 667         final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 668
 669         if (actionForNode == Action.DELETE && !isProgrammed) {
 670             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 671                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 672             return;
 673         }
 674         if (actionForNode == Action.ADD && isProgrammed) {
 675             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 676                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 677             return;
 678         }
 679
 680         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 681                                                        macAddress, ipStr, actionForNode);
 682         if (status.isSuccess()) {
 683             // Update cache
 684             if (actionForNode == Action.ADD) {
 685                 l3ForwardingCache.add(cacheKey);
 686             } else {
 687                 l3ForwardingCache.remove(cacheKey);
 688             }
 689         }
 690     }
 691
 692     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 693                                              String macAddress,
 694                                              String address,
 695                                              Action actionForNode) {
 696         Status status;
 697         try {
 698             InetAddress inetAddress = InetAddress.getByName(address);
 699             status = l3ForwardingProvider == null ?
 700                      new Status(StatusCode.SUCCESS) :
 701                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 702                                                                       inetAddress, macAddress, actionForNode);
 703         } catch (UnknownHostException e) {
 704             status = new Status(StatusCode.BADREQUEST);
 705         }
 706
 707         if (status.isSuccess()) {
 708             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 709                          l3ForwardingProvider == null ? "skipped" : "programmed",
 710                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 711         } else {
 712             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 713                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 714         }
 715         return status;
 716     }
 717
 718     // --
 719
 720     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 721                                                        Boolean isDelete) {
 722         Preconditions.checkNotNull(destNeutronRouterInterface);
 723
 724         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 725         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 726         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 727         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 728         final NeutronNetwork neutronNetwork = subnet != null ?
 729                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 730         final String destinationSegmentationId = neutronNetwork != null ?
 731                                                  neutronNetwork.getProviderSegmentationID() : null;
 732         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 733         final String cidr = subnet != null ? subnet.getCidr() : null;
 734         final int mask = getMaskLenFromCidr(cidr);
 735
 736         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 737                      destNeutronRouterInterface, isDelete);
 738
 739         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 740         // the local cache
 741         if (neutronNetwork != null) {
 742             if (macAddress == null || macAddress.isEmpty()) {
 743                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 744             }
 745             if (ipList == null || ipList.isEmpty()) {
 746                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 747             }
 748         }
 749
 750         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 751             cidr == null || cidr.isEmpty() ||
 752             macAddress == null || macAddress.isEmpty() ||
 753             ipList == null || ipList.isEmpty()) {
 754             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 755                          destinationSegmentationId, cidr, macAddress, ipList);
 756             // done: go no further w/out all the info needed...
 757             return;
 758         }
 759
 760         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 761
 762         // Keep cache for finding router's mac from network uuid -- add
 763         //
 764         if (! isDelete) {
 765             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 766             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 767             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 768         }
 769
 770         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 771         if (nodes.isEmpty()) {
 772             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 773         }
 774         for (Node node : nodes) {
 775             final Long dpid = getDpidForIntegrationBridge(node);
 776             if (dpid == null) {
 777                 continue;
 778             }
 779
 780             for (Neutron_IPs neutronIP : ipList) {
 781                 final String ipStr = neutronIP.getIpAddress();
 782                 if (ipStr.isEmpty()) {
 783                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 784                             node.getNodeId().getValue(), ipStr);
 785                     continue;
 786                 }
 787
 788                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 789                 //
 790                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 791                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 792                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 793                                                               neutronNetwork, destinationSegmentationId,
 794                                                               macAddress, ipStr, mask, actionForNode,
 795                                                               true /*isReflexsive*/);
 796                 }
 797
 798                 if (! isExternal) {
 799                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 800                             actionForNode);
 801                 }
 802                 // Enable ARP responder by default, because router interface needs to be responded always.
 803                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 804             }
 805
 806             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 807             // for the external neutron networks.
 808             //
 809             {
 810                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 811                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 812             }
 813         }
 814
 815         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
 816     }
 817
 818     private void programFlowForNetworkFromExternal(final Node node,
 819                                                    final Long dpid,
 820                                                    final String destinationSegmentationId,
 821                                                    final String dstMacAddress,
 822                                                    final String destIpStr,
 823                                                    final int destMask,
 824                                                    final Action actionForNode) {
 825         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 826                 dstMacAddress, destIpStr, destMask, actionForNode);
 827     }
 828
 829     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 830                                                            final Long dpid,
 831                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 832                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 833                                                            final NeutronNetwork dstNeutronNetwork,
 834                                                            final String destinationSegmentationId,
 835                                                            final String dstMacAddress,
 836                                                            final String destIpStr,
 837                                                            final int destMask,
 838                                                            final Action actionForNode,
 839                                                            Boolean isReflexsive) {
 840         Preconditions.checkNotNull(srcNeutronRouterInterface);
 841         Preconditions.checkNotNull(dstNeutronRouterInterface);
 842
 843         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 844         if (sourceSubnetId == null) {
 845             LOG.error("Could not get provider Subnet ID from router interface {}",
 846                          srcNeutronRouterInterface.getID());
 847             return;
 848         }
 849
 850         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 851         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 852         if (sourceNetworkId == null) {
 853             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 854             return;
 855         }
 856
 857         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 858         if (sourceNetwork == null) {
 859             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 860             return;
 861         }
 862
 863         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 864             // Isolate subnets from different tenants within the same router
 865             return;
 866         }
 867         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 868         if (sourceSegmentationId == null) {
 869             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 870             return;
 871         }
 872         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 873             // Skip 'self'
 874             return;
 875         }
 876
 877         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 878                                      dstMacAddress, destIpStr, destMask, actionForNode);
 879
 880         // Flip roles src->dst; dst->src
 881         if (isReflexsive) {
 882             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 883             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 884             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 885             final String cidr2 = sourceSubnet.getCidr();
 886             final int mask2 = getMaskLenFromCidr(cidr2);
 887
 888             if (cidr2 == null || cidr2.isEmpty() ||
 889                 macAddress2 == null || macAddress2.isEmpty() ||
 890                 ipList2 == null || ipList2.isEmpty()) {
 891                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 892                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 893                 // done: go no further w/out all the info needed...
 894                 return;
 895             }
 896
 897             for (Neutron_IPs neutronIP2 : ipList2) {
 898                 final String ipStr2 = neutronIP2.getIpAddress();
 899                 if (ipStr2.isEmpty()) {
 900                     continue;
 901                 }
 902                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 903                                                           srcNeutronRouterInterface,
 904                                                           sourceNetwork, sourceSegmentationId,
 905                                                           macAddress2, ipStr2, mask2, actionForNode,
 906                                                           false /*isReflexsive*/);
 907             }
 908         }
 909     }
 910
 911     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 912                                               String destinationSegmentationId,
 913                                               String macAddress, String ipStr, int mask,
 914                                               Action actionForNode) {
 915         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 916         // will look at desired action for node.
 917         //
 918         final String cacheKey = node.getNodeId().getValue() + ":" +
 919                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 920                                 ipStr + "/" + Integer.toString(mask);
 921         final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 922
 923         if (actionForNode == Action.DELETE && !isProgrammed) {
 924             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 925                          " action {} is already done",
 926                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 927                          macAddress, ipStr, mask, actionForNode);
 928             return;
 929         }
 930         if (actionForNode == Action.ADD && isProgrammed) {
 931             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 932                          " action {} is already done",
 933                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 934                          macAddress, ipStr, mask, actionForNode);
 935             return;
 936         }
 937
 938         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 939                                                           macAddress, ipStr, mask, actionForNode);
 940         if (status.isSuccess()) {
 941             // Update cache
 942             if (actionForNode == Action.ADD) {
 943                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 944                 routerInterfacesCache.add(cacheKey);
 945             } else {
 946                 // TODO: multiTenantAwareRouter.removeInterface(...);
 947                 routerInterfacesCache.remove(cacheKey);
 948             }
 949         }
 950     }
 951
 952     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 953                                                 String destinationSegmentationId,
 954                                                 String macAddress,
 955                                                 String address, int mask,
 956                                                 Action actionForNode) {
 957         Status status;
 958         try {
 959             InetAddress inetAddress = InetAddress.getByName(address);
 960             status = routingProvider == null ?
 961                      new Status(StatusCode.SUCCESS) :
 962                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 963                                                             macAddress, inetAddress, mask, actionForNode);
 964         } catch (UnknownHostException e) {
 965             status = new Status(StatusCode.BADREQUEST);
 966         }
 967
 968         if (status.isSuccess()) {
 969             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 970                          routingProvider == null ? "skipped" : "programmed",
 971                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 972                          actionForNode);
 973         } else {
 974             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 975                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 976                          actionForNode, status);
 977         }
 978         return status;
 979     }
 980
 981     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 982                                            String macAddress, String ipStr,
 983                                            Action action) {
 984         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 985         // will look at desired action for node.
 986         //
 987         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 988         final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 989
 990         if (action == Action.DELETE && !isProgrammed) {
 991             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 992                     dpid, segOrOfPort, macAddress, ipStr, action);
 993             return true;
 994         }
 995         if (action == Action.ADD && isProgrammed) {
 996             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 997                     dpid, segOrOfPort, macAddress, ipStr, action);
 998             return true;
 999         }
1000
1001         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1002         if (status.isSuccess()) {
1003             // Update cache
1004             if (action == Action.ADD) {
1005                 staticArpEntryCache.add(cacheKey);
1006             } else {
1007                 staticArpEntryCache.remove(cacheKey);
1008             }
1009             return true;
1010         }
1011         return false;
1012     }
1013
1014     private Status programStaticArpStage2(Long dpid,
1015                                           String segOrOfPort,
1016                                           String macAddress,
1017                                           String address,
1018                                           Action action) {
1019         Status status;
1020         try {
1021             InetAddress inetAddress = InetAddress.getByName(address);
1022             status = arpProvider == null ?
1023                      new Status(StatusCode.SUCCESS) :
1024                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1025                                                        macAddress, inetAddress, action);
1026         } catch (UnknownHostException e) {
1027             status = new Status(StatusCode.BADREQUEST);
1028         }
1029
1030         if (status.isSuccess()) {
1031             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1032                          arpProvider == null ? "skipped" : "programmed",
1033                          macAddress, address, dpid, segOrOfPort, action);
1034         } else {
1035             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1036                          macAddress, address, dpid, segOrOfPort, action, status);
1037         }
1038         return status;
1039     }
1040
1041     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1042                                                   String matchAddress, String rewriteAddress,
1043                                                   Action action) {
1044         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1045         // will look at desired action for node.
1046         //
1047         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1048         final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1049
1050         if (action == Action.DELETE && !isProgrammed) {
1051             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1052                     " action {} is already done",
1053                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1054             return true;
1055         }
1056         if (action == Action.ADD && isProgrammed) {
1057             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1058                     " action is already done",
1059                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1060             return true;
1061         }
1062
1063         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1064                 rewriteAddress, action);
1065         if (status.isSuccess()) {
1066             // Update cache
1067             if (action == Action.ADD) {
1068                 inboundIpRewriteCache.add(cacheKey);
1069             } else {
1070                 inboundIpRewriteCache.remove(cacheKey);
1071             }
1072             return true;
1073         }
1074         return false;
1075     }
1076
1077     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1078                                                  String matchAddress, String rewriteAddress,
1079                                                  Action action) {
1080         Status status;
1081         try {
1082             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1083             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1084             status = inboundNatProvider == null ?
1085                     new Status(StatusCode.SUCCESS) :
1086                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1087                             inetMatchAddress, inetRewriteAddress,
1088                             action);
1089         } catch (UnknownHostException e) {
1090             status = new Status(StatusCode.BADREQUEST);
1091         }
1092
1093         if (status.isSuccess()) {
1094             final boolean isSkipped = inboundNatProvider == null;
1095             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1096                     isSkipped ? "skipped" : "programmed",
1097                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1098         } else {
1099             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1100                          " status:{}",
1101                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1102                     status);
1103         }
1104         return status;
1105     }
1106
1107     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1108                                                  Action actionForRewriteExclusion) {
1109         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1110         // will look at desired action for node.
1111         //
1112         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1113         final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1114
1115         if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
1116             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1117                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1118             return;
1119         }
1120         if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
1121             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1122                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1123             return;
1124         }
1125
1126         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1127                                                              actionForRewriteExclusion);
1128         if (status.isSuccess()) {
1129             // Update cache
1130             if (actionForRewriteExclusion == Action.ADD) {
1131                     outboundIpRewriteExclusionCache.add(cacheKey);
1132             } else {
1133                     outboundIpRewriteExclusionCache.remove(cacheKey);
1134             }
1135         }
1136     }
1137
1138     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1139                                                    Action actionForNode) {
1140         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1141                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1142
1143         if (status.isSuccess()) {
1144             final boolean isSkipped = outboundNatProvider == null;
1145             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1146                          isSkipped ? "skipped" : "programmed",
1147                          cidr, node.getNodeId().getValue(), actionForNode);
1148         } else {
1149             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1150                          cidr, node.getNodeId().getValue(), actionForNode, status);
1151         }
1152         return status;
1153     }
1154
1155     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1156         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1157         // will look at desired action for node.
1158         //
1159         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1160         final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1161
1162         if (action == Action.DELETE && !isProgrammed) {
1163             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1164                          "is already done",
1165                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1166             return;
1167         }
1168         if (action == Action.ADD && isProgrammed) {
1169             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1170                          "is already done",
1171                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1172             return;
1173         }
1174
1175         Status status = this.programOutboundIpRewriteStage2(fid, action);
1176         if (status.isSuccess()) {
1177             // Update cache
1178             if (action == Action.ADD) {
1179                 outboundIpRewriteCache.add(cacheKey);
1180             } else {
1181                 outboundIpRewriteCache.remove(cacheKey);
1182             }
1183         }
1184     }
1185
1186     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1187         Status status;
1188         try {
1189             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1190             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1191             status = outboundNatProvider == null ?
1192                     new Status(StatusCode.SUCCESS) :
1193                     outboundNatProvider.programIpRewriteRule(
1194                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1195                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1196         } catch (UnknownHostException e) {
1197             status = new Status(StatusCode.BADREQUEST);
1198         }
1199
1200         if (status.isSuccess()) {
1201             final boolean isSkipped = outboundNatProvider == null;
1202             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1203                             " action {}",
1204                          isSkipped ? "skipped" : "programmed",
1205                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1206         } else {
1207             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1208                          " action {} status:{}",
1209                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1210         }
1211         return status;
1212     }
1213
1214     private int getMaskLenFromCidr(String cidr) {
1215         if (cidr == null) {
1216             return 0;
1217         }
1218         String[] splits = cidr.split("/");
1219         if (splits.length != 2) {
1220             return 0;
1221         }
1222
1223         int result;
1224         try {
1225             result = Integer.parseInt(splits[1].trim());
1226         } catch (NumberFormatException nfe) {
1227             result = 0;
1228         }
1229         return result;
1230     }
1231
1232     private Long getDpidForIntegrationBridge(Node node) {
1233         // Check if node is integration bridge; and only then return its dpid
1234         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1235             return southbound.getDataPathId(node);
1236         }
1237         return null;
1238     }
1239
1240     private Long getDpidForExternalBridge(Node node) {
1241         // Check if node is integration bridge; and only then return its dpid
1242         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1243             return southbound.getDataPathId(node);
1244         }
1245         return null;
1246     }
1247
1248     private Node getExternalBridgeNode(){
1249         //Pickup the first node that has external bridge (br-ex).
1250         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1251         //the external network is reachable from every br-ex
1252         // TODO: Consider other deployment scenario, and thing of better solution.
1253         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1254         for(Node node : allBridges){
1255             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1256                 return node;
1257             }
1258         }
1259         return null;
1260     }
1261
1262     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1263         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1264             String subnetUUID = neutronIPs.getSubnetUUID();
1265             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1266             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1267                 return extSubnet;
1268             }
1269             if (extSubnet == null) {
1270                 // TODO: when subnet is created, try again.
1271                 LOG.debug("subnet {} in not found", subnetUUID);
1272              }
1273         }
1274         return null;
1275     }
1276
1277      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1278          /*
1279           *  Fix for 4277
1280           *  Remove the router cache only after deleting the neutron
1281           *  port l3 flows.
1282           */
1283          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1284
1285          if (neutronPort != null) {
1286              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1287              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1288              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1289          }
1290      }
1291
1292     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1293
1294         Preconditions.checkNotNull(node);
1295         Preconditions.checkNotNull(gatewayPort);
1296         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1297
1298         if(externalNetwork != null){
1299             if(externalNetwork.isRouterExternal()){
1300                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1301
1302                 // TODO: address IPv6 case.
1303                 if (externalSubnet != null &&
1304                     externalSubnet.getIpVersion() == 4) {
1305                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1306                     ListenableFuture<MacAddress> gatewayMacAddress =
1307                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1308                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1309                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1310                                                              new MacAddress(gatewayPort.getMacAddress()),
1311                                                              true);
1312                     if(gatewayMacAddress != null){
1313                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1314                             @Override
1315                             public void onSuccess(MacAddress result) {
1316                                 if(result != null){
1317                                     if(!result.getValue().equals(externalRouterMac)){
1318                                         updateExternalRouterMac(result.getValue());
1319                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1320                                     }
1321                                 }else{
1322                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1323                                 }
1324                             }
1325
1326                             @Override
1327                             public void onFailure(Throwable t) {
1328                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1329                             }
1330                         }, gatewayMacResolverPool);
1331                     }
1332                 } else {
1333                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1334                 }
1335             }
1336         }else{
1337             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1338         }
1339     }
1340
1341     /**
1342      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1343      *
1344      * @param ofPort the OF port number
1345      * @return the string with encoded OF port (example format "OFPort|999")
1346      */
1347     public static String encodeExcplicitOFPort(Long ofPort) {
1348         return "OFPort|" + ofPort.toString();
1349     }
1350
1351     @Override
1352     public void setDependencies(ServiceReference serviceReference) {
1353         tenantNetworkManager =
1354                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1355         configurationService =
1356                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1357         arpProvider =
1358                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1359         inboundNatProvider =
1360                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1361         outboundNatProvider =
1362                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1363         routingProvider =
1364                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1365         l3ForwardingProvider =
1366                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1367         nodeCacheManager =
1368                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1369         southbound =
1370                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1371         gatewayMacResolver =
1372                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1373         initL3AdapterMembers();
1374     }
1375
1376     @Override
1377     public void setDependencies(Object impl) {
1378         if (impl instanceof INeutronNetworkCRUD) {
1379             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1380         } else if (impl instanceof INeutronPortCRUD) {
1381             neutronPortCache = (INeutronPortCRUD)impl;
1382         } else if (impl instanceof INeutronSubnetCRUD) {
1383             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1384         } else if (impl instanceof ArpProvider) {
1385             arpProvider = (ArpProvider)impl;
1386         } else if (impl instanceof InboundNatProvider) {
1387             inboundNatProvider = (InboundNatProvider)impl;
1388         } else if (impl instanceof OutboundNatProvider) {
1389             outboundNatProvider = (OutboundNatProvider)impl;
1390         } else if (impl instanceof RoutingProvider) {
1391             routingProvider = (RoutingProvider)impl;
1392         } else if (impl instanceof L3ForwardingProvider) {
1393             l3ForwardingProvider = (L3ForwardingProvider)impl;
1394         }else if (impl instanceof GatewayMacResolver) {
1395             gatewayMacResolver = (GatewayMacResolver)impl;
1396         }
1397     }
1398 }