openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import org.apache.commons.lang3.tuple.ImmutablePair;
  12 import org.apache.commons.lang3.tuple.Pair;
  13 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  14 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  15 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  16 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  17 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  18 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  19 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  20 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  21 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  22 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  23 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  28 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  29 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  30 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  31 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  32
  33 import com.google.common.base.Preconditions;
  34 import com.google.common.util.concurrent.FutureCallback;
  35 import com.google.common.util.concurrent.Futures;
  36 import com.google.common.util.concurrent.ListenableFuture;
  37
  38 import org.osgi.framework.ServiceReference;
  39 import org.slf4j.Logger;
  40 import org.slf4j.LoggerFactory;
  41
  42 import java.net.InetAddress;
  43 import java.net.UnknownHostException;
  44 import java.util.HashMap;
  45 import java.util.HashSet;
  46 import java.util.ArrayList;
  47 import java.util.Iterator;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.Set;
  51 import java.util.concurrent.ExecutorService;
  52 import java.util.concurrent.Executors;
  53
  54 /**
  55  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  56  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  57  * as well as the multi-tenant router forwarding provider.
  58  */
  59 public class NeutronL3Adapter implements ConfigInterface {
  60     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  61
  62     // The implementation for each of these services is resolved by the OSGi Service Manager
  63     private volatile ConfigurationService configurationService;
  64     private volatile TenantNetworkManager tenantNetworkManager;
  65     private volatile NodeCacheManager nodeCacheManager;
  66     private volatile INeutronNetworkCRUD neutronNetworkCache;
  67     private volatile INeutronSubnetCRUD neutronSubnetCache;
  68     private volatile INeutronPortCRUD neutronPortCache;
  69     private volatile L3ForwardingProvider l3ForwardingProvider;
  70     private volatile InboundNatProvider inboundNatProvider;
  71     private volatile OutboundNatProvider outboundNatProvider;
  72     private volatile ArpProvider arpProvider;
  73     private volatile RoutingProvider routingProvider;
  74     private volatile GatewayMacResolver gatewayMacResolver;
  75     private volatile SecurityServicesManager securityServicesManager;
  76
  77     private class FloatIpData {
  78         // br-int of node where floating ip is associated with tenant port
  79         private final Long dpid;
  80         // patch port in br-int used to reach br-ex
  81         private final Long ofPort;
  82         // segmentation id of the net where fixed ip is instantiated
  83         private final String segId;
  84         // mac address assigned to neutron port of floating ip
  85         private final String macAddress;
  86         private final String floatingIpAddress;
  87         // ip address given to tenant vm
  88         private final String fixedIpAddress;
  89         private final String neutronRouterMac;
  90
  91         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  92                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  93             this.dpid = dpid;
  94             this.ofPort = ofPort;
  95             this.segId = segId;
  96             this.macAddress = macAddress;
  97             this.floatingIpAddress = floatingIpAddress;
  98             this.fixedIpAddress = fixedIpAddress;
  99             this.neutronRouterMac = neutronRouterMac;
 100         }
 101     }
 102
 103     private Set<String> inboundIpRewriteCache;
 104     private Set<String> outboundIpRewriteCache;
 105     private Set<String> outboundIpRewriteExclusionCache;
 106     private Set<String> routerInterfacesCache;
 107     private Set<String> staticArpEntryCache;
 108     private Set<String> l3ForwardingCache;
 109     private Map<String, String> networkIdToRouterMacCache;
 110     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 111     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 112     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 113     private Map<String, FloatIpData> floatIpDataMapCache;
 114     private String externalRouterMac;
 115     private Boolean enabled = false;
 116     private Boolean flgDistributedARPEnabled = true;
 117     private Southbound southbound;
 118     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 119
 120     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 121     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 122     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 123     private static final String OWNER_FLOATING_IP = "network:floatingip";
 124     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 125
 126     public NeutronL3Adapter() {
 127         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 128     }
 129
 130     private void initL3AdapterMembers() {
 131         Preconditions.checkNotNull(configurationService);
 132
 133         if (configurationService.isL3ForwardingEnabled()) {
 134             this.inboundIpRewriteCache = new HashSet<>();
 135             this.outboundIpRewriteCache = new HashSet<>();
 136             this.outboundIpRewriteExclusionCache = new HashSet<>();
 137             this.routerInterfacesCache = new HashSet<>();
 138             this.staticArpEntryCache = new HashSet<>();
 139             this.l3ForwardingCache = new HashSet<>();
 140             this.networkIdToRouterMacCache = new HashMap<>();
 141             this.networkIdToRouterIpListCache = new HashMap<>();
 142             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 143             this.neutronPortToDpIdCache = new HashMap<>();
 144             this.floatIpDataMapCache = new HashMap<>();
 145
 146             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 147             if (this.externalRouterMac == null) {
 148                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 149             }
 150
 151             this.enabled = true;
 152             LOG.info("OVSDB L3 forwarding is enabled");
 153             if (configurationService.isDistributedArpDisabled()) {
 154                 this.flgDistributedARPEnabled = false;
 155                 LOG.info("Distributed ARP responder is disabled");
 156             } else {
 157                 LOG.debug("Distributed ARP responder is enabled");
 158             }
 159         } else {
 160             LOG.debug("OVSDB L3 forwarding is disabled");
 161         }
 162     }
 163
 164     //
 165     // Callbacks from OVSDB's northbound handlers
 166     //
 167
 168     /**
 169      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 170      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 171      * subscribe so that the mac address associated to the gateway ip address is available.
 172      *
 173      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 174      */
 175     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 176         Preconditions.checkNotNull(externalRouterMacUpdate);
 177
 178         flushExistingIpRewrite();
 179         this.externalRouterMac = externalRouterMacUpdate;
 180         rebuildExistingIpRewrite();
 181     }
 182
 183     /**
 184      * Process the event.
 185      *
 186      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 187      * @param subnet An instance of NeutronSubnet object.
 188      */
 189     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 190         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 191     }
 192
 193     /**
 194      * Process the port event as a router interface event.
 195      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 196      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 197      *
 198      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 199      * @param neutronPort An instance of NeutronPort object.
 200      */
 201     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 202         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 203
 204         this.processSecurityGroupUpdate(neutronPort);
 205         if (!this.enabled) {
 206             return;
 207         }
 208
 209         final boolean isDelete = action == Action.DELETE;
 210
 211         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 212             if(!isDelete){
 213                 Node externalBridgeNode = getExternalBridgeNode();
 214                 if(externalBridgeNode != null){
 215                     LOG.info("Port {} is network router gateway interface, "
 216                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 217                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 218                 }else{
 219                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 220                 }
 221             }else{
 222                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 223
 224                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 225                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 226                     // TODO support IPv6
 227                     if (externalSubnet != null &&
 228                             externalSubnet.getIpVersion() == 4) {
 229                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 230                     }
 231                 }
 232             }
 233         }
 234
 235         // Treat the port event as a router interface event if the port belongs to router. This is a
 236         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 237         //
 238         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 239             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 240
 241             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 242                 NeutronRouter_Interface neutronRouterInterface =
 243                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 244                 // id of router interface to be same as subnet
 245                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 246                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 247
 248                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 249             }
 250         } else {
 251             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 252             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 253             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 254             // there.
 255             //
 256             if (!isDelete && neutronPort.getFixedIPs() != null) {
 257                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 258                     NeutronRouter_Interface neutronRouterInterface =
 259                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 260                     if (neutronRouterInterface != null) {
 261                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 262                     }
 263                 }
 264             }
 265             this.updateL3ForNeutronPort(neutronPort, isDelete);
 266         }
 267     }
 268
 269     /**
 270      * Process the event.
 271      *
 272      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 273      * @param neutronRouter An instance of NeutronRouter object.
 274      */
 275     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 276         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 277     }
 278
 279     /**
 280      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 281      * delete the ports on the same subnet.
 282      *
 283      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 284      * @param neutronRouter An instance of NeutronRouter object.
 285      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 286      */
 287     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 288                                                   final NeutronRouter_Interface neutronRouterInterface,
 289                                                   Action action) {
 290         LOG.debug("Router interface {} got event {}. Subnet {}",
 291                      neutronRouterInterface.getPortUUID(),
 292                      action,
 293                      neutronRouterInterface.getSubnetUUID());
 294         if (!this.enabled) {
 295             return;
 296         }
 297
 298         final boolean isDelete = action == Action.DELETE;
 299
 300         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 301
 302         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 303         // see if they are affected by l3
 304         //
 305         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 306             boolean currPortShouldBeDeleted = false;
 307             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 308             if (isDelete) {
 309                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 310                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 311                         currPortShouldBeDeleted = true;
 312                         break;
 313                     }
 314                 }
 315             }
 316             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 317         }
 318
 319         if (isDelete) {
 320             /*
 321              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 322              */
 323             this.cleanupRouterCache(neutronRouterInterface);
 324         }
 325     }
 326
 327     /**
 328      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 329      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 330      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 331      * way around) on OpenFlow Table 100.
 332      *
 333      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 334      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 335      */
 336     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 337                                              Action actionIn) {
 338         Preconditions.checkNotNull(neutronFloatingIP);
 339
 340         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 341                 neutronFloatingIP.getFixedIPAddress(),
 342                 neutronFloatingIP.getFloatingIPAddress(),
 343                 neutronFloatingIP.getFloatingNetworkUUID());
 344         if (!this.enabled) {
 345             return;
 346         }
 347
 348         Action action;
 349
 350         // Consider action to be delete if getFixedIPAddress is null
 351         //
 352         if (neutronFloatingIP.getFixedIPAddress() == null) {
 353             action = Action.DELETE;
 354         } else {
 355             action = actionIn;
 356         }
 357
 358         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 359
 360         if (action != Action.DELETE) {
 361             // must be first, as it updates floatIpDataMapCache
 362             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 363
 364             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 365             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 366         } else {
 367             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 368             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 369
 370             // must be last, as it updates floatIpDataMapCache
 371             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 372         }
 373     }
 374
 375     /**
 376      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 377      * ip, otherwise for newer one.
 378      */
 379     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 380         Preconditions.checkNotNull(neutronFloatingIP);
 381
 382         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 383         if (fid == null) {
 384             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 385                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 386             return;
 387         }
 388         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 389                                       action);
 390     }
 391
 392     /**
 393      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 394      * ip, otherwise for newer one.
 395      */
 396     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 397         Preconditions.checkNotNull(neutronFloatingIP);
 398
 399         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 400         if (fid == null) {
 401             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 402                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 403             return;
 404         }
 405         programOutboundIpRewriteStage1(fid, action);
 406     }
 407
 408     private void flushExistingIpRewrite() {
 409         for (FloatIpData fid : floatIpDataMapCache.values()) {
 410             programOutboundIpRewriteStage1(fid, Action.DELETE);
 411         }
 412     }
 413
 414     private void rebuildExistingIpRewrite() {
 415         for (FloatIpData fid : floatIpDataMapCache.values()) {
 416             programOutboundIpRewriteStage1(fid, Action.ADD);
 417         }
 418     }
 419
 420     /**
 421      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 422      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 423      * requests.
 424      */
 425     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 426         Preconditions.checkNotNull(neutronFloatingIP);
 427         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 428         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 429
 430         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 431             LOG.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 432                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 433             return;
 434         }
 435
 436         // find bridge Node where floating ip is configured by looking up cache for its port
 437         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 438         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 439         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 440         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 441         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 442         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 443
 444         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 445         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 446                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 447         final String providerSegmentationId = tenantNeutronNetwork != null ?
 448                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 449         final String neutronRouterMac = tenantNeutronNetwork != null ?
 450                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 451
 452         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 453                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 454                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 455                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 456             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 457                     fixedIpAddress,
 458                     floatingIpAddress,
 459                     neutronPortForFloatIp,
 460                     neutronTenantPortUuid,
 461                     providerSegmentationId,
 462                     floatingIpMac,
 463                     neutronRouterMac);
 464             return;
 465         }
 466
 467         // get ofport for patch port in br-int
 468         final Long dpId = nodeIfPair.getLeft();
 469         final Long ofPort = findOFPortForExtPatch(dpId);
 470         if (ofPort == null) {
 471             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 472                     dpId);
 473             return;
 474         }
 475
 476         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 477         //
 478         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 479                 Action.ADD)) {
 480             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 481                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 482             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 483             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 484                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 485                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 486         }
 487     }
 488
 489     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 490         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 491         if (floatIpData == null) {
 492             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 493             return;
 494         }
 495
 496         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 497                 floatIpData.floatingIpAddress, Action.DELETE)) {
 498             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 499             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 500                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 501         }
 502     }
 503
 504     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 505         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 506             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 507                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 508                 return neutronPort;
 509             }
 510         }
 511         return null;
 512     }
 513
 514     private Long findOFPortForExtPatch(Long dpId) {
 515         final String brInt = configurationService.getIntegrationBridgeName();
 516         final String brExt = configurationService.getExternalBridgeName();
 517         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 518
 519         Preconditions.checkNotNull(dpId);
 520         Preconditions.checkNotNull(portNameInt);
 521
 522         final long dpidPrimitive = dpId;
 523         for (Node node : nodeCacheManager.getBridgeNodes()) {
 524             if (dpidPrimitive == southbound.getDataPathId(node)) {
 525                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 526                         southbound.getTerminationPointOfBridge(node, portNameInt);
 527                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 528             }
 529         }
 530         return null;
 531     }
 532
 533     /**
 534      * Process the event.
 535      *
 536      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 537      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 538      */
 539     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 540         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 541     }
 542
 543     //
 544     // Callbacks from OVSDB's southbound handler
 545     //
 546     /**
 547      * Process the event.
 548      *
 549      * @param bridgeNode An instance of Node object.
 550      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 551      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 552      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 553      * object.
 554      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 555      */
 556     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 557                                      final NeutronNetwork neutronNetwork, Action action) {
 558         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 559                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 560         if (!this.enabled) {
 561             return;
 562         }
 563
 564         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 565         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 566         final Uuid interfaceUuid = intf.getInterfaceUuid();
 567
 568         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 569                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 570
 571         if (neutronPort != null) {
 572             final String neutronPortUuid = neutronPort.getPortUUID();
 573
 574             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 575                     dpId != null && interfaceUuid != null) {
 576                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 577             }
 578
 579             handleNeutronPortEvent(neutronPort, action);
 580         }
 581
 582         if (action == Action.DELETE && interfaceUuid != null) {
 583             handleInterfaceEventDelete(intf, dpId);
 584         }
 585     }
 586
 587     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 588         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 589         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 590                 neutronPortUuid, dpId, interfaceUuid.getValue());
 591     }
 592
 593     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 594         // Remove entry from neutronPortToDpIdCache based on interface uuid
 595         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 596             final String currPortUuid = entry.getKey();
 597             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 598                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 599                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 600                 neutronPortToDpIdCache.remove(currPortUuid);
 601                 break;
 602             }
 603         }
 604     }
 605
 606     //
 607     // Internal helpers
 608     //
 609     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 610
 611         final String networkUUID = neutronPort.getNetworkUUID();
 612         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 613
 614         // If there is no router interface handling the networkUUID, we are done
 615         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 616             return;
 617         }
 618
 619         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 620         // router interface are handled via handleNeutronRouterInterfaceEvent.
 621         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 622             return;
 623         }
 624
 625         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 626         final String providerSegmentationId = neutronNetwork != null ?
 627                                               neutronNetwork.getProviderSegmentationID() : null;
 628         final String tenantMac = neutronPort.getMacAddress();
 629
 630         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 631             tenantMac == null || tenantMac.isEmpty()) {
 632             // done: go no further w/out all the info needed...
 633             return;
 634         }
 635
 636         final Action action = isDelete ? Action.DELETE : Action.ADD;
 637         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 638         if (nodes.isEmpty()) {
 639             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 640         }
 641         for (Node node : nodes) {
 642             final Long dpid = getDpidForIntegrationBridge(node);
 643             if (dpid == null) {
 644                 continue;
 645             }
 646
 647             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 648                 final String tenantIpStr = neutronIP.getIpAddress();
 649                 if (tenantIpStr.isEmpty()) {
 650                     continue;
 651                 }
 652
 653                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 654                 // still needed when routing to subnets non-local to node (bug 2076).
 655                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 656
 657                 // Configure distributed ARP responder
 658                 if (flgDistributedARPEnabled) {
 659                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 660                 }
 661             }
 662         }
 663     }
 664
 665     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 666         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 667         /**
 668          * Get updated data and original data for the the changed. Identify the security groups that got
 669          * added and removed and call the appropriate providers for updating the flows.
 670          */
 671         try {
 672             NeutronPort originalPort = neutronPort.getOriginalPort();
 673             if (null == originalPort) {
 674                 LOG.debug("processSecurityGroupUpdate: originalport is empty");
 675                 return;
 676             }
 677             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 678                                                                             neutronPort.getOriginalPort());
 679             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 680                                                                               neutronPort);
 681
 682             if (null != addedGroup && !addedGroup.isEmpty()) {
 683                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 684             }
 685             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 686                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 687             }
 688
 689         } catch (Exception e) {
 690             LOG.error("Exception in processSecurityGroupUpdate", e);
 691         }
 692     }
 693
 694     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 695         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 696         ArrayList<NeutronSecurityGroup> list1 = new ArrayList<NeutronSecurityGroup>(port1.getSecurityGroups());
 697         ArrayList<NeutronSecurityGroup> list2 = new ArrayList<NeutronSecurityGroup>(port2.getSecurityGroups());
 698         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 699             NeutronSecurityGroup securityGroup1 = iterator.next();
 700             for (NeutronSecurityGroup securityGroup2 :list2) {
 701                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 702                     iterator.remove();
 703                 }
 704             }
 705         }
 706         return list1;
 707     }
 708
 709     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 710                                            String macAddress, String ipStr,
 711                                            Action actionForNode) {
 712         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 713         // will look at desired action for node.
 714
 715         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 716         final boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 717
 718         if (actionForNode == Action.DELETE && !isProgrammed) {
 719             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 720                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 721             return;
 722         }
 723         if (actionForNode == Action.ADD && isProgrammed) {
 724             LOG.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 725                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 726             return;
 727         }
 728
 729         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 730                                                        macAddress, ipStr, actionForNode);
 731         if (status.isSuccess()) {
 732             // Update cache
 733             if (actionForNode == Action.ADD) {
 734                 l3ForwardingCache.add(cacheKey);
 735             } else {
 736                 l3ForwardingCache.remove(cacheKey);
 737             }
 738         }
 739     }
 740
 741     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 742                                              String macAddress,
 743                                              String address,
 744                                              Action actionForNode) {
 745         Status status;
 746         try {
 747             InetAddress inetAddress = InetAddress.getByName(address);
 748             status = l3ForwardingProvider == null ?
 749                      new Status(StatusCode.SUCCESS) :
 750                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 751                                                                       inetAddress, macAddress, actionForNode);
 752         } catch (UnknownHostException e) {
 753             status = new Status(StatusCode.BADREQUEST);
 754         }
 755
 756         if (status.isSuccess()) {
 757             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 758                          l3ForwardingProvider == null ? "skipped" : "programmed",
 759                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 760         } else {
 761             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 762                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 763         }
 764         return status;
 765     }
 766
 767     // --
 768
 769     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 770                                                        Boolean isDelete) {
 771         Preconditions.checkNotNull(destNeutronRouterInterface);
 772
 773         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 774         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 775         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 776         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 777         final NeutronNetwork neutronNetwork = subnet != null ?
 778                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 779         final String destinationSegmentationId = neutronNetwork != null ?
 780                                                  neutronNetwork.getProviderSegmentationID() : null;
 781         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 782         final String cidr = subnet != null ? subnet.getCidr() : null;
 783         final int mask = getMaskLenFromCidr(cidr);
 784
 785         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 786                      destNeutronRouterInterface, isDelete);
 787
 788         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 789         // the local cache
 790         if (neutronNetwork != null) {
 791             if (macAddress == null || macAddress.isEmpty()) {
 792                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 793             }
 794             if (ipList == null || ipList.isEmpty()) {
 795                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 796             }
 797         }
 798
 799         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 800             cidr == null || cidr.isEmpty() ||
 801             macAddress == null || macAddress.isEmpty() ||
 802             ipList == null || ipList.isEmpty()) {
 803             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 804                          destinationSegmentationId, cidr, macAddress, ipList);
 805             // done: go no further w/out all the info needed...
 806             return;
 807         }
 808
 809         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 810
 811         // Keep cache for finding router's mac from network uuid -- add
 812         //
 813         if (! isDelete) {
 814             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 815             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 816             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 817         }
 818
 819         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 820         if (nodes.isEmpty()) {
 821             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 822         }
 823         for (Node node : nodes) {
 824             final Long dpid = getDpidForIntegrationBridge(node);
 825             if (dpid == null) {
 826                 continue;
 827             }
 828
 829             for (Neutron_IPs neutronIP : ipList) {
 830                 final String ipStr = neutronIP.getIpAddress();
 831                 if (ipStr.isEmpty()) {
 832                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 833                             node.getNodeId().getValue(), ipStr);
 834                     continue;
 835                 }
 836
 837                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 838                 //
 839                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 840                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 841                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 842                                                               neutronNetwork, destinationSegmentationId,
 843                                                               macAddress, ipStr, mask, actionForNode,
 844                                                               true /*isReflexsive*/);
 845                 }
 846
 847                 if (! isExternal) {
 848                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 849                             actionForNode);
 850                 }
 851                 // Enable ARP responder by default, because router interface needs to be responded always.
 852                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 853             }
 854
 855             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 856             // for the external neutron networks.
 857             //
 858             {
 859                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 860                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 861             }
 862         }
 863
 864         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
 865     }
 866
 867     private void programFlowForNetworkFromExternal(final Node node,
 868                                                    final Long dpid,
 869                                                    final String destinationSegmentationId,
 870                                                    final String dstMacAddress,
 871                                                    final String destIpStr,
 872                                                    final int destMask,
 873                                                    final Action actionForNode) {
 874         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 875                 dstMacAddress, destIpStr, destMask, actionForNode);
 876     }
 877
 878     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 879                                                            final Long dpid,
 880                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 881                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 882                                                            final NeutronNetwork dstNeutronNetwork,
 883                                                            final String destinationSegmentationId,
 884                                                            final String dstMacAddress,
 885                                                            final String destIpStr,
 886                                                            final int destMask,
 887                                                            final Action actionForNode,
 888                                                            Boolean isReflexsive) {
 889         Preconditions.checkNotNull(srcNeutronRouterInterface);
 890         Preconditions.checkNotNull(dstNeutronRouterInterface);
 891
 892         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 893         if (sourceSubnetId == null) {
 894             LOG.error("Could not get provider Subnet ID from router interface {}",
 895                          srcNeutronRouterInterface.getID());
 896             return;
 897         }
 898
 899         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 900         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 901         if (sourceNetworkId == null) {
 902             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 903             return;
 904         }
 905
 906         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 907         if (sourceNetwork == null) {
 908             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 909             return;
 910         }
 911
 912         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 913             // Isolate subnets from different tenants within the same router
 914             return;
 915         }
 916         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 917         if (sourceSegmentationId == null) {
 918             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 919             return;
 920         }
 921         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 922             // Skip 'self'
 923             return;
 924         }
 925
 926         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 927                                      dstMacAddress, destIpStr, destMask, actionForNode);
 928
 929         // Flip roles src->dst; dst->src
 930         if (isReflexsive) {
 931             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 932             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 933             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 934             final String cidr2 = sourceSubnet.getCidr();
 935             final int mask2 = getMaskLenFromCidr(cidr2);
 936
 937             if (cidr2 == null || cidr2.isEmpty() ||
 938                 macAddress2 == null || macAddress2.isEmpty() ||
 939                 ipList2 == null || ipList2.isEmpty()) {
 940                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 941                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 942                 // done: go no further w/out all the info needed...
 943                 return;
 944             }
 945
 946             for (Neutron_IPs neutronIP2 : ipList2) {
 947                 final String ipStr2 = neutronIP2.getIpAddress();
 948                 if (ipStr2.isEmpty()) {
 949                     continue;
 950                 }
 951                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 952                                                           srcNeutronRouterInterface,
 953                                                           sourceNetwork, sourceSegmentationId,
 954                                                           macAddress2, ipStr2, mask2, actionForNode,
 955                                                           false /*isReflexsive*/);
 956             }
 957         }
 958     }
 959
 960     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 961                                               String destinationSegmentationId,
 962                                               String macAddress, String ipStr, int mask,
 963                                               Action actionForNode) {
 964         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 965         // will look at desired action for node.
 966         //
 967         final String cacheKey = node.getNodeId().getValue() + ":" +
 968                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 969                                 ipStr + "/" + Integer.toString(mask);
 970         final boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 971
 972         if (actionForNode == Action.DELETE && !isProgrammed) {
 973             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 974                          " action {} is already done",
 975                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 976                          macAddress, ipStr, mask, actionForNode);
 977             return;
 978         }
 979         if (actionForNode == Action.ADD && isProgrammed) {
 980             LOG.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 981                          " action {} is already done",
 982                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 983                          macAddress, ipStr, mask, actionForNode);
 984             return;
 985         }
 986
 987         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 988                                                           macAddress, ipStr, mask, actionForNode);
 989         if (status.isSuccess()) {
 990             // Update cache
 991             if (actionForNode == Action.ADD) {
 992                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 993                 routerInterfacesCache.add(cacheKey);
 994             } else {
 995                 // TODO: multiTenantAwareRouter.removeInterface(...);
 996                 routerInterfacesCache.remove(cacheKey);
 997             }
 998         }
 999     }
1000
1001     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1002                                                 String destinationSegmentationId,
1003                                                 String macAddress,
1004                                                 String address, int mask,
1005                                                 Action actionForNode) {
1006         Status status;
1007         try {
1008             InetAddress inetAddress = InetAddress.getByName(address);
1009             status = routingProvider == null ?
1010                      new Status(StatusCode.SUCCESS) :
1011                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1012                                                             macAddress, inetAddress, mask, actionForNode);
1013         } catch (UnknownHostException e) {
1014             status = new Status(StatusCode.BADREQUEST);
1015         }
1016
1017         if (status.isSuccess()) {
1018             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1019                          routingProvider == null ? "skipped" : "programmed",
1020                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1021                          actionForNode);
1022         } else {
1023             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1024                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1025                          actionForNode, status);
1026         }
1027         return status;
1028     }
1029
1030     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1031                                            String macAddress, String ipStr,
1032                                            Action action) {
1033         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1034         // will look at desired action for node.
1035         //
1036         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
1037         final boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
1038
1039         if (action == Action.DELETE && !isProgrammed) {
1040             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1041                     dpid, segOrOfPort, macAddress, ipStr, action);
1042             return true;
1043         }
1044         if (action == Action.ADD && isProgrammed) {
1045             LOG.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1046                     dpid, segOrOfPort, macAddress, ipStr, action);
1047             return true;
1048         }
1049
1050         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1051         if (status.isSuccess()) {
1052             // Update cache
1053             if (action == Action.ADD) {
1054                 staticArpEntryCache.add(cacheKey);
1055             } else {
1056                 staticArpEntryCache.remove(cacheKey);
1057             }
1058             return true;
1059         }
1060         return false;
1061     }
1062
1063     private Status programStaticArpStage2(Long dpid,
1064                                           String segOrOfPort,
1065                                           String macAddress,
1066                                           String address,
1067                                           Action action) {
1068         Status status;
1069         try {
1070             InetAddress inetAddress = InetAddress.getByName(address);
1071             status = arpProvider == null ?
1072                      new Status(StatusCode.SUCCESS) :
1073                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1074                                                        macAddress, inetAddress, action);
1075         } catch (UnknownHostException e) {
1076             status = new Status(StatusCode.BADREQUEST);
1077         }
1078
1079         if (status.isSuccess()) {
1080             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1081                          arpProvider == null ? "skipped" : "programmed",
1082                          macAddress, address, dpid, segOrOfPort, action);
1083         } else {
1084             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1085                          macAddress, address, dpid, segOrOfPort, action, status);
1086         }
1087         return status;
1088     }
1089
1090     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1091                                                   String matchAddress, String rewriteAddress,
1092                                                   Action action) {
1093         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1094         // will look at desired action for node.
1095         //
1096         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
1097         final boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
1098
1099         if (action == Action.DELETE && !isProgrammed) {
1100             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1101                     " action {} is already done",
1102                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1103             return true;
1104         }
1105         if (action == Action.ADD && isProgrammed) {
1106             LOG.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1107                     " action is already done",
1108                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1109             return true;
1110         }
1111
1112         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1113                 rewriteAddress, action);
1114         if (status.isSuccess()) {
1115             // Update cache
1116             if (action == Action.ADD) {
1117                 inboundIpRewriteCache.add(cacheKey);
1118             } else {
1119                 inboundIpRewriteCache.remove(cacheKey);
1120             }
1121             return true;
1122         }
1123         return false;
1124     }
1125
1126     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1127                                                  String matchAddress, String rewriteAddress,
1128                                                  Action action) {
1129         Status status;
1130         try {
1131             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1132             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1133             status = inboundNatProvider == null ?
1134                     new Status(StatusCode.SUCCESS) :
1135                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1136                             inetMatchAddress, inetRewriteAddress,
1137                             action);
1138         } catch (UnknownHostException e) {
1139             status = new Status(StatusCode.BADREQUEST);
1140         }
1141
1142         if (status.isSuccess()) {
1143             final boolean isSkipped = inboundNatProvider == null;
1144             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1145                     isSkipped ? "skipped" : "programmed",
1146                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1147         } else {
1148             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1149                          " status:{}",
1150                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1151                     status);
1152         }
1153         return status;
1154     }
1155
1156     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1157                                                  Action actionForRewriteExclusion) {
1158         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1159         // will look at desired action for node.
1160         //
1161         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1162         final boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1163
1164         if (actionForRewriteExclusion == Action.DELETE && !isProgrammed) {
1165             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1166                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1167             return;
1168         }
1169         if (actionForRewriteExclusion == Action.ADD && isProgrammed) {
1170             LOG.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1171                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1172             return;
1173         }
1174
1175         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1176                                                              actionForRewriteExclusion);
1177         if (status.isSuccess()) {
1178             // Update cache
1179             if (actionForRewriteExclusion == Action.ADD) {
1180                     outboundIpRewriteExclusionCache.add(cacheKey);
1181             } else {
1182                     outboundIpRewriteExclusionCache.remove(cacheKey);
1183             }
1184         }
1185     }
1186
1187     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1188                                                    Action actionForNode) {
1189         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1190                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1191
1192         if (status.isSuccess()) {
1193             final boolean isSkipped = outboundNatProvider == null;
1194             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1195                          isSkipped ? "skipped" : "programmed",
1196                          cidr, node.getNodeId().getValue(), actionForNode);
1197         } else {
1198             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1199                          cidr, node.getNodeId().getValue(), actionForNode, status);
1200         }
1201         return status;
1202     }
1203
1204     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1205         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1206         // will look at desired action for node.
1207         //
1208         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1209         final boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1210
1211         if (action == Action.DELETE && !isProgrammed) {
1212             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1213                          "is already done",
1214                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1215             return;
1216         }
1217         if (action == Action.ADD && isProgrammed) {
1218             LOG.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1219                          "is already done",
1220                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1221             return;
1222         }
1223
1224         Status status = this.programOutboundIpRewriteStage2(fid, action);
1225         if (status.isSuccess()) {
1226             // Update cache
1227             if (action == Action.ADD) {
1228                 outboundIpRewriteCache.add(cacheKey);
1229             } else {
1230                 outboundIpRewriteCache.remove(cacheKey);
1231             }
1232         }
1233     }
1234
1235     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1236         Status status;
1237         try {
1238             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1239             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1240             status = outboundNatProvider == null ?
1241                     new Status(StatusCode.SUCCESS) :
1242                     outboundNatProvider.programIpRewriteRule(
1243                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1244                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1245         } catch (UnknownHostException e) {
1246             status = new Status(StatusCode.BADREQUEST);
1247         }
1248
1249         if (status.isSuccess()) {
1250             final boolean isSkipped = outboundNatProvider == null;
1251             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1252                             " action {}",
1253                          isSkipped ? "skipped" : "programmed",
1254                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1255         } else {
1256             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1257                          " action {} status:{}",
1258                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1259         }
1260         return status;
1261     }
1262
1263     private int getMaskLenFromCidr(String cidr) {
1264         if (cidr == null) {
1265             return 0;
1266         }
1267         String[] splits = cidr.split("/");
1268         if (splits.length != 2) {
1269             return 0;
1270         }
1271
1272         int result;
1273         try {
1274             result = Integer.parseInt(splits[1].trim());
1275         } catch (NumberFormatException nfe) {
1276             result = 0;
1277         }
1278         return result;
1279     }
1280
1281     private Long getDpidForIntegrationBridge(Node node) {
1282         // Check if node is integration bridge; and only then return its dpid
1283         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1284             return southbound.getDataPathId(node);
1285         }
1286         return null;
1287     }
1288
1289     private Long getDpidForExternalBridge(Node node) {
1290         // Check if node is integration bridge; and only then return its dpid
1291         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1292             return southbound.getDataPathId(node);
1293         }
1294         return null;
1295     }
1296
1297     private Node getExternalBridgeNode(){
1298         //Pickup the first node that has external bridge (br-ex).
1299         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1300         //the external network is reachable from every br-ex
1301         // TODO: Consider other deployment scenario, and thing of better solution.
1302         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1303         for(Node node : allBridges){
1304             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1305                 return node;
1306             }
1307         }
1308         return null;
1309     }
1310
1311     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1312         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1313             String subnetUUID = neutronIPs.getSubnetUUID();
1314             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1315             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1316                 return extSubnet;
1317             }
1318             if (extSubnet == null) {
1319                 // TODO: when subnet is created, try again.
1320                 LOG.debug("subnet {} in not found", subnetUUID);
1321              }
1322         }
1323         return null;
1324     }
1325
1326      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1327          /*
1328           *  Fix for 4277
1329           *  Remove the router cache only after deleting the neutron
1330           *  port l3 flows.
1331           */
1332          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1333
1334          if (neutronPort != null) {
1335              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1336              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1337              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1338          }
1339      }
1340
1341     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1342
1343         Preconditions.checkNotNull(node);
1344         Preconditions.checkNotNull(gatewayPort);
1345         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1346
1347         if(externalNetwork != null){
1348             if(externalNetwork.isRouterExternal()){
1349                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1350
1351                 // TODO: address IPv6 case.
1352                 if (externalSubnet != null &&
1353                     externalSubnet.getIpVersion() == 4) {
1354                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1355                     ListenableFuture<MacAddress> gatewayMacAddress =
1356                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1357                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1358                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1359                                                              new MacAddress(gatewayPort.getMacAddress()),
1360                                                              true);
1361                     if(gatewayMacAddress != null){
1362                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1363                             @Override
1364                             public void onSuccess(MacAddress result) {
1365                                 if(result != null){
1366                                     if(!result.getValue().equals(externalRouterMac)){
1367                                         updateExternalRouterMac(result.getValue());
1368                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1369                                     }
1370                                 }else{
1371                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1372                                 }
1373                             }
1374
1375                             @Override
1376                             public void onFailure(Throwable t) {
1377                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1378                             }
1379                         }, gatewayMacResolverPool);
1380                     }
1381                 } else {
1382                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1383                 }
1384             }
1385         }else{
1386             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1387         }
1388     }
1389
1390     /**
1391      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1392      *
1393      * @param ofPort the OF port number
1394      * @return the string with encoded OF port (example format "OFPort|999")
1395      */
1396     public static String encodeExcplicitOFPort(Long ofPort) {
1397         return "OFPort|" + ofPort.toString();
1398     }
1399
1400     @Override
1401     public void setDependencies(ServiceReference serviceReference) {
1402         tenantNetworkManager =
1403                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1404         configurationService =
1405                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1406         arpProvider =
1407                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1408         inboundNatProvider =
1409                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1410         outboundNatProvider =
1411                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1412         routingProvider =
1413                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1414         l3ForwardingProvider =
1415                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1416         nodeCacheManager =
1417                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1418         southbound =
1419                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1420         gatewayMacResolver =
1421                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1422         securityServicesManager =
1423                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1424         initL3AdapterMembers();
1425     }
1426
1427     @Override
1428     public void setDependencies(Object impl) {
1429         if (impl instanceof INeutronNetworkCRUD) {
1430             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1431         } else if (impl instanceof INeutronPortCRUD) {
1432             neutronPortCache = (INeutronPortCRUD)impl;
1433         } else if (impl instanceof INeutronSubnetCRUD) {
1434             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1435         } else if (impl instanceof ArpProvider) {
1436             arpProvider = (ArpProvider)impl;
1437         } else if (impl instanceof InboundNatProvider) {
1438             inboundNatProvider = (InboundNatProvider)impl;
1439         } else if (impl instanceof OutboundNatProvider) {
1440             outboundNatProvider = (OutboundNatProvider)impl;
1441         } else if (impl instanceof RoutingProvider) {
1442             routingProvider = (RoutingProvider)impl;
1443         } else if (impl instanceof L3ForwardingProvider) {
1444             l3ForwardingProvider = (L3ForwardingProvider)impl;
1445         }else if (impl instanceof GatewayMacResolver) {
1446             gatewayMacResolver = (GatewayMacResolver)impl;
1447         }
1448     }
1449 }