openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import java.net.InetAddress;
  12 import java.net.UnknownHostException;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.ExecutorService;
  20 import java.util.concurrent.Executors;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  49 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  50 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  51 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  52 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  53 import org.opendaylight.ovsdb.utils.mdsal.utils.NeutronModelsDataStoreHelper;
  54 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  63 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  64 import org.osgi.framework.ServiceReference;
  65 import org.slf4j.Logger;
  66 import org.slf4j.LoggerFactory;
  67
  68 import com.google.common.base.Preconditions;
  69 import com.google.common.util.concurrent.FutureCallback;
  70 import com.google.common.util.concurrent.Futures;
  71 import com.google.common.util.concurrent.ListenableFuture;
  72
  73 /**
  74  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  75  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  76  * as well as the multi-tenant router forwarding provider.
  77  */
  78 public class NeutronL3Adapter implements ConfigInterface {
  79     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  80
  81     // The implementation for each of these services is resolved by the OSGi Service Manager
  82     private volatile ConfigurationService configurationService;
  83     private volatile TenantNetworkManager tenantNetworkManager;
  84     private volatile NodeCacheManager nodeCacheManager;
  85     private volatile INeutronNetworkCRUD neutronNetworkCache;
  86     private volatile INeutronSubnetCRUD neutronSubnetCache;
  87     private volatile INeutronPortCRUD neutronPortCache;
  88     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  89     private volatile L3ForwardingProvider l3ForwardingProvider;
  90     private volatile InboundNatProvider inboundNatProvider;
  91     private volatile OutboundNatProvider outboundNatProvider;
  92     private volatile ArpProvider arpProvider;
  93     private volatile RoutingProvider routingProvider;
  94     private volatile GatewayMacResolver gatewayMacResolver;
  95     private volatile SecurityServicesManager securityServicesManager;
  96
  97     private class FloatIpData {
  98         // br-int of node where floating ip is associated with tenant port
  99         private final Long dpid;
 100         // patch port in br-int used to reach br-ex
 101         private final Long ofPort;
 102         // segmentation id of the net where fixed ip is instantiated
 103         private final String segId;
 104         // mac address assigned to neutron port of floating ip
 105         private final String macAddress;
 106         private final String floatingIpAddress;
 107         // ip address given to tenant vm
 108         private final String fixedIpAddress;
 109         private final String neutronRouterMac;
 110
 111         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 112                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 113             this.dpid = dpid;
 114             this.ofPort = ofPort;
 115             this.segId = segId;
 116             this.macAddress = macAddress;
 117             this.floatingIpAddress = floatingIpAddress;
 118             this.fixedIpAddress = fixedIpAddress;
 119             this.neutronRouterMac = neutronRouterMac;
 120         }
 121     }
 122
 123     private Map<String, String> networkIdToRouterMacCache;
 124     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 125     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 126
 127     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 128     private Map<String, FloatIpData> floatIpDataMapCache;
 129
 130     private String externalRouterMac;
 131     private Boolean enabled = false;
 132     private Boolean flgDistributedARPEnabled = true;
 133     private Boolean isCachePopulationDone = false;
 134     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 135
 136     private Southbound southbound;
 137     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 138
 139     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 140     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 141     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 142     private static final String OWNER_FLOATING_IP = "network:floatingip";
 143     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 144
 145     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 146         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 147         this.neutronModelsDataStoreHelper = neutronHelper;
 148     }
 149
 150     private void initL3AdapterMembers() {
 151         Preconditions.checkNotNull(configurationService);
 152
 153         if (configurationService.isL3ForwardingEnabled()) {
 154             this.networkIdToRouterMacCache = new HashMap<>();
 155             this.networkIdToRouterIpListCache = new HashMap<>();
 156             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 157             this.neutronPortToDpIdCache = new HashMap<>();
 158             this.floatIpDataMapCache = new HashMap<>();
 159
 160             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 161             if (this.externalRouterMac == null) {
 162                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 163             }
 164
 165             this.enabled = true;
 166             LOG.info("OVSDB L3 forwarding is enabled");
 167             if (configurationService.isDistributedArpDisabled()) {
 168                 this.flgDistributedARPEnabled = false;
 169                 LOG.info("Distributed ARP responder is disabled");
 170             } else {
 171                 LOG.debug("Distributed ARP responder is enabled");
 172             }
 173         } else {
 174             LOG.debug("OVSDB L3 forwarding is disabled");
 175         }
 176     }
 177
 178
 179     private void populateL3ForwardingCaches() {
 180         if (!this.enabled) {
 181             return;
 182         }
 183         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 184                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 185             return;
 186         }
 187         this.isCachePopulationDone = true;
 188         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 189         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 190         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 191         if(routers != null && routers.getRouter() != null && ports != null) {
 192             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 193             for( Router router : routers.getRouter()) {
 194                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 195                 if(!ports.getPort().isEmpty()) {
 196                     for( Port port : ports.getPort()) {
 197                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 198                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 199                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 200                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 201                                     , port.getMacAddress());
 202
 203                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 204                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 205                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 206                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 207                         }
 208                     }
 209                 }else {
 210                     LOG.warn("L3 Cache Population :Did not find any port information " +
 211                             "in config Data Store for router {}",router);
 212                 }
 213             }
 214         }
 215         LOG.debug("NetVirt L3 caches population is done");
 216     }
 217
 218     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 219         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 220             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 221             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 222             for(Node bridge : bridges) {
 223                 List<OvsdbTerminationPointAugmentation> interfaces
 224                         = southbound.extractTerminationPointAugmentations(bridge);
 225                 if(interfaces != null && !interfaces.isEmpty()) {
 226                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 227                             interfaces.size(), bridge.getNodeId());
 228                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 229                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 230                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 231                             Long dpId = getDpidForIntegrationBridge(bridge);
 232                             Uuid interfaceUuid = intf.getInterfaceUuid();
 233                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 234                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 235                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 236                             break;
 237                         }
 238                     }
 239                 }
 240             }
 241         }
 242         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 243     }
 244
 245     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 246         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 247         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 248         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 249             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 250                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 251                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 252                             "Fetching from data store.",neutronFloatingIP.getID());
 253                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 254                 }
 255             }
 256         }
 257         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 258         return floatIpDataMapCache.values();
 259     }
 260     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 261         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 262                 "floating id {} ",neutronFloatingId);
 263         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 264             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 265             if (neutronFloatingIP == null) {
 266                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 267                 return null;
 268             }
 269             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 270             NeutronPort neutronPortForFloatIp = null;
 271             for (NeutronPort neutronPort : neutronPorts) {
 272                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 273                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 274                     neutronPortForFloatIp = neutronPort;
 275                     break;
 276                 }
 277             }
 278
 279             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 280             if(neutronTenantPortUuid == null) {
 281                 return null;
 282             }
 283             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 284             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 285             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 286             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 287
 288             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 289             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 290                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 291             String providerSegmentationId = tenantNeutronNetwork != null ?
 292                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 293             String neutronRouterMac = tenantNeutronNetwork != null ?
 294                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 295
 296             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 297                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 298                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 299                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 300                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 301                                 "seg {} mac {} rtrMac {}",
 302                         fixedIpAddress,
 303                         floatingIpAddress,
 304                         neutronPortForFloatIp,
 305                         neutronTenantPortUuid,
 306                         providerSegmentationId,
 307                         floatingIpMac,
 308                         neutronRouterMac);
 309
 310                 return null;
 311             }
 312
 313             // get ofport for patch port in br-int
 314             final Long dpId = nodeIfPair.getLeft();
 315             final Long ofPort = findOFPortForExtPatch(dpId);
 316             if (ofPort == null) {
 317                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 318                                 "to connect floating ip to external bridge. dpid {}",
 319                         dpId);
 320                 return null;
 321             }
 322
 323             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 324                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 325             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 326
 327         }
 328         return floatIpDataMapCache.get(neutronFloatingId);
 329     }
 330     /**
 331      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 332      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 333      * subscribe so that the mac address associated to the gateway ip address is available.
 334      *
 335      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 336      */
 337     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 338         Preconditions.checkNotNull(externalRouterMacUpdate);
 339
 340         flushExistingIpRewrite();
 341         this.externalRouterMac = externalRouterMacUpdate;
 342         rebuildExistingIpRewrite();
 343     }
 344
 345     /**
 346      * Process the event.
 347      *
 348      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 349      * @param subnet An instance of NeutronSubnet object.
 350      */
 351     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 352         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 353     }
 354
 355     /**
 356      * Process the port event as a router interface event.
 357      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 358      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 359      *
 360      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 361      * @param neutronPort An instance of NeutronPort object.
 362      */
 363     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 364         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 365
 366         this.processSecurityGroupUpdate(neutronPort);
 367         if (!this.enabled) {
 368             return;
 369         }
 370
 371         final boolean isDelete = action == Action.DELETE;
 372
 373         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 374             if(!isDelete){
 375                 Node externalBridgeNode = getExternalBridgeNode();
 376                 if(externalBridgeNode != null){
 377                     LOG.info("Port {} is network router gateway interface, "
 378                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 379                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 380                 }else{
 381                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 382                 }
 383             }else{
 384                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 385
 386                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 387                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 388                     // TODO support IPv6
 389                     if (externalSubnet != null &&
 390                             externalSubnet.getIpVersion() == 4) {
 391                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 392                     }
 393                 }
 394             }
 395         }
 396
 397         // Treat the port event as a router interface event if the port belongs to router. This is a
 398         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 399         //
 400         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 401             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 402
 403             if (neutronPort.getFixedIPs() != null) {
 404                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 405                     NeutronRouter_Interface neutronRouterInterface =
 406                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 407                     // id of router interface to be same as subnet
 408                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 409                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 410
 411                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 412                 }
 413             }
 414         } else {
 415             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 416             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 417             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 418             // there.
 419             //
 420             if (!isDelete && neutronPort.getFixedIPs() != null) {
 421                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 422                     NeutronRouter_Interface neutronRouterInterface =
 423                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 424                     if (neutronRouterInterface != null) {
 425                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 426                     }
 427                 }
 428             }
 429             this.updateL3ForNeutronPort(neutronPort, isDelete);
 430         }
 431     }
 432
 433     /**
 434      * Process the event.
 435      *
 436      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 437      * @param neutronRouter An instance of NeutronRouter object.
 438      */
 439     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 440         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 441     }
 442
 443     /**
 444      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 445      * delete the ports on the same subnet.
 446      *
 447      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 448      * @param neutronRouter An instance of NeutronRouter object.
 449      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 450      */
 451     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 452                                                   final NeutronRouter_Interface neutronRouterInterface,
 453                                                   Action action) {
 454         LOG.debug("Router interface {} got event {}. Subnet {}",
 455                      neutronRouterInterface.getPortUUID(),
 456                      action,
 457                      neutronRouterInterface.getSubnetUUID());
 458         if (!this.enabled) {
 459             return;
 460         }
 461
 462         final boolean isDelete = action == Action.DELETE;
 463
 464         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 465
 466         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 467         // see if they are affected by l3
 468         //
 469         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 470             boolean currPortShouldBeDeleted = false;
 471             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 472             if (isDelete) {
 473                 if (neutronPort.getFixedIPs() != null) {
 474                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 475                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 476                             currPortShouldBeDeleted = true;
 477                             break;
 478                         }
 479                     }
 480                 }
 481             }
 482             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 483         }
 484
 485         if (isDelete) {
 486             /*
 487              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 488              */
 489             this.cleanupRouterCache(neutronRouterInterface);
 490         }
 491     }
 492
 493     /**
 494      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 495      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 496      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 497      * way around) on OpenFlow Table 100.
 498      *
 499      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 500      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 501      */
 502     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 503                                              Action actionIn) {
 504         Preconditions.checkNotNull(neutronFloatingIP);
 505
 506         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 507                 neutronFloatingIP.getFixedIPAddress(),
 508                 neutronFloatingIP.getFloatingIPAddress(),
 509                 neutronFloatingIP.getFloatingNetworkUUID());
 510         if (!this.enabled) {
 511             return;
 512         }
 513
 514         Action action;
 515
 516         // Consider action to be delete if getFixedIPAddress is null
 517         //
 518         if (neutronFloatingIP.getFixedIPAddress() == null) {
 519             action = Action.DELETE;
 520         } else {
 521             action = actionIn;
 522         }
 523
 524         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 525
 526         if (action != Action.DELETE) {
 527             // must be first, as it updates floatIpDataMapCache
 528             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 529
 530             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 531             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 532         } else {
 533             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 534             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 535
 536             // must be last, as it updates floatIpDataMapCache
 537             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 538         }
 539     }
 540
 541     /**
 542      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 543      * ip, otherwise for newer one.
 544      */
 545     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 546         Preconditions.checkNotNull(neutronFloatingIP);
 547
 548         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 549         if (fid == null) {
 550             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 551                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 552             return;
 553         }
 554         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 555                                       action);
 556     }
 557
 558     /**
 559      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 560      * ip, otherwise for newer one.
 561      */
 562     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 563         Preconditions.checkNotNull(neutronFloatingIP);
 564
 565         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 566         if (fid == null) {
 567             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 568                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 569             return;
 570         }
 571         programOutboundIpRewriteStage1(fid, action);
 572     }
 573
 574     private void flushExistingIpRewrite() {
 575         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 576             programOutboundIpRewriteStage1(fid, Action.DELETE);
 577         }
 578     }
 579
 580     private void rebuildExistingIpRewrite() {
 581         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 582             programOutboundIpRewriteStage1(fid, Action.ADD);
 583         }
 584     }
 585
 586     /**
 587      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 588      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 589      * requests.
 590      */
 591     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 592         Preconditions.checkNotNull(neutronFloatingIP);
 593         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 594         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 595
 596         // find bridge Node where floating ip is configured by looking up cache for its port
 597         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 598         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 599         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 600         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 601         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 602         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 603
 604         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 605         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 606                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 607         final String providerSegmentationId = tenantNeutronNetwork != null ?
 608                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 609         final String neutronRouterMac = tenantNeutronNetwork != null ?
 610                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 611
 612         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 613                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 614                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 615                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 616             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 617                     fixedIpAddress,
 618                     floatingIpAddress,
 619                     neutronPortForFloatIp,
 620                     neutronTenantPortUuid,
 621                     providerSegmentationId,
 622                     floatingIpMac,
 623                     neutronRouterMac);
 624             return;
 625         }
 626
 627         // get ofport for patch port in br-int
 628         final Long dpId = nodeIfPair.getLeft();
 629         final Long ofPort = findOFPortForExtPatch(dpId);
 630         if (ofPort == null) {
 631             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 632                     dpId);
 633             return;
 634         }
 635
 636         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 637         //
 638         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 639                 Action.ADD)) {
 640             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 641                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 642             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 643             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 644                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 645                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 646         }
 647     }
 648
 649     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 650         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 651         if (floatIpData == null) {
 652             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 653             return;
 654         }
 655
 656         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 657                 floatIpData.floatingIpAddress, Action.DELETE)) {
 658             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 659             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 660                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 661         }
 662     }
 663
 664     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 665         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 666             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 667                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 668                 return neutronPort;
 669             }
 670         }
 671         return null;
 672     }
 673
 674     private Long findOFPortForExtPatch(Long dpId) {
 675         final String brInt = configurationService.getIntegrationBridgeName();
 676         final String brExt = configurationService.getExternalBridgeName();
 677         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 678
 679         Preconditions.checkNotNull(dpId);
 680         Preconditions.checkNotNull(portNameInt);
 681
 682         final long dpidPrimitive = dpId;
 683         for (Node node : nodeCacheManager.getBridgeNodes()) {
 684             if (dpidPrimitive == southbound.getDataPathId(node)) {
 685                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 686                         southbound.getTerminationPointOfBridge(node, portNameInt);
 687                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 688             }
 689         }
 690         return null;
 691     }
 692
 693     /**
 694      * Process the event.
 695      *
 696      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 697      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 698      */
 699     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 700         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 701     }
 702
 703     //
 704     // Callbacks from OVSDB's southbound handler
 705     //
 706     /**
 707      * Process the event.
 708      *
 709      * @param bridgeNode An instance of Node object.
 710      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 711      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 712      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 713      * object.
 714      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 715      */
 716     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 717                                      final NeutronNetwork neutronNetwork, Action action) {
 718         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 719                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 720         if (!this.enabled) {
 721             return;
 722         }
 723
 724         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 725         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 726         final Uuid interfaceUuid = intf.getInterfaceUuid();
 727
 728         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 729                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 730
 731         if (neutronPort != null) {
 732             final String neutronPortUuid = neutronPort.getPortUUID();
 733
 734             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 735                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 736             }
 737
 738             handleNeutronPortEvent(neutronPort, action);
 739         }
 740
 741         if (action == Action.DELETE && interfaceUuid != null) {
 742             handleInterfaceEventDelete(intf, dpId);
 743         }
 744     }
 745
 746     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 747         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 748         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 749                 neutronPortUuid, dpId, interfaceUuid.getValue());
 750     }
 751
 752     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 753         // Remove entry from neutronPortToDpIdCache based on interface uuid
 754         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 755             final String currPortUuid = entry.getKey();
 756             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 757                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 758                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 759                 neutronPortToDpIdCache.remove(currPortUuid);
 760                 break;
 761             }
 762         }
 763     }
 764
 765     //
 766     // Internal helpers
 767     //
 768     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 769
 770         final String networkUUID = neutronPort.getNetworkUUID();
 771         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 772
 773         // If there is no router interface handling the networkUUID, we are done
 774         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 775             return;
 776         }
 777
 778         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 779         // router interface are handled via handleNeutronRouterInterfaceEvent.
 780         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 781             return;
 782         }
 783
 784         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 785         final String providerSegmentationId = neutronNetwork != null ?
 786                                               neutronNetwork.getProviderSegmentationID() : null;
 787         final String tenantMac = neutronPort.getMacAddress();
 788
 789         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 790             tenantMac == null || tenantMac.isEmpty()) {
 791             // done: go no further w/out all the info needed...
 792             return;
 793         }
 794
 795         final Action action = isDelete ? Action.DELETE : Action.ADD;
 796         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 797         if (nodes.isEmpty()) {
 798             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 799         }
 800         for (Node node : nodes) {
 801             final Long dpid = getDpidForIntegrationBridge(node);
 802             if (dpid == null) {
 803                 continue;
 804             }
 805             if (neutronPort.getFixedIPs() == null) {
 806                 continue;
 807             }
 808             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 809                 final String tenantIpStr = neutronIP.getIpAddress();
 810                 if (tenantIpStr.isEmpty()) {
 811                     continue;
 812                 }
 813
 814                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 815                 // still needed when routing to subnets non-local to node (bug 2076).
 816                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 817
 818                 // Configure distributed ARP responder
 819                 if (flgDistributedARPEnabled) {
 820                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 821                 }
 822             }
 823         }
 824     }
 825
 826     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 827         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 828         /**
 829          * Get updated data and original data for the the changed. Identify the security groups that got
 830          * added and removed and call the appropriate providers for updating the flows.
 831          */
 832         try {
 833             NeutronPort originalPort = neutronPort.getOriginalPort();
 834             if (null == originalPort) {
 835                 LOG.debug("processSecurityGroupUpdate: originalport is empty");
 836                 return;
 837             }
 838             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 839                                                                             neutronPort.getOriginalPort());
 840             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 841                                                                               neutronPort);
 842
 843             if (null != addedGroup && !addedGroup.isEmpty()) {
 844                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 845             }
 846             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 847                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 848             }
 849
 850         } catch (Exception e) {
 851             LOG.error("Exception in processSecurityGroupUpdate", e);
 852         }
 853     }
 854
 855     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 856         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 857         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 858         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 859         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 860             NeutronSecurityGroup securityGroup1 = iterator.next();
 861             for (NeutronSecurityGroup securityGroup2 :list2) {
 862                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 863                     iterator.remove();
 864                 }
 865             }
 866         }
 867         return list1;
 868     }
 869
 870     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 871                                            String macAddress, String ipStr,
 872                                            Action actionForNode) {
 873         if (actionForNode == Action.DELETE) {
 874             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 875                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 876         }
 877         if (actionForNode == Action.ADD) {
 878             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 879                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 880         }
 881
 882         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 883                                                        macAddress, ipStr, actionForNode);
 884     }
 885
 886     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 887                                              String macAddress,
 888                                              String address,
 889                                              Action actionForNode) {
 890         Status status;
 891         try {
 892             InetAddress inetAddress = InetAddress.getByName(address);
 893             status = l3ForwardingProvider == null ?
 894                      new Status(StatusCode.SUCCESS) :
 895                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 896                                                                       inetAddress, macAddress, actionForNode);
 897         } catch (UnknownHostException e) {
 898             status = new Status(StatusCode.BADREQUEST);
 899         }
 900
 901         if (status.isSuccess()) {
 902             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 903                          l3ForwardingProvider == null ? "skipped" : "programmed",
 904                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 905         } else {
 906             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 907                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 908         }
 909         return status;
 910     }
 911
 912     // --
 913
 914     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 915                                                        Boolean isDelete) {
 916         Preconditions.checkNotNull(destNeutronRouterInterface);
 917
 918         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 919         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 920         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 921         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 922         final NeutronNetwork neutronNetwork = subnet != null ?
 923                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 924         final String destinationSegmentationId = neutronNetwork != null ?
 925                                                  neutronNetwork.getProviderSegmentationID() : null;
 926         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 927         final String cidr = subnet != null ? subnet.getCidr() : null;
 928         final int mask = getMaskLenFromCidr(cidr);
 929
 930         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 931                      destNeutronRouterInterface, isDelete);
 932
 933         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 934         // the local cache
 935         if (neutronNetwork != null) {
 936             if (macAddress == null || macAddress.isEmpty()) {
 937                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 938             }
 939             if (ipList == null || ipList.isEmpty()) {
 940                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 941             }
 942         }
 943
 944         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 945             cidr == null || cidr.isEmpty() ||
 946             macAddress == null || macAddress.isEmpty() ||
 947             ipList == null || ipList.isEmpty()) {
 948             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 949                          destinationSegmentationId, cidr, macAddress, ipList);
 950             // done: go no further w/out all the info needed...
 951             return;
 952         }
 953
 954         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 955
 956         // Keep cache for finding router's mac from network uuid -- add
 957         //
 958         if (! isDelete) {
 959             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 960             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 961             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 962         }
 963
 964         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 965         if (nodes.isEmpty()) {
 966             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 967         }
 968         for (Node node : nodes) {
 969             final Long dpid = getDpidForIntegrationBridge(node);
 970             if (dpid == null) {
 971                 continue;
 972             }
 973
 974             for (Neutron_IPs neutronIP : ipList) {
 975                 final String ipStr = neutronIP.getIpAddress();
 976                 if (ipStr.isEmpty()) {
 977                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 978                             node.getNodeId().getValue(), ipStr);
 979                     continue;
 980                 }
 981
 982                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 983                 //
 984                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 985                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 986                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 987                                                               neutronNetwork, destinationSegmentationId,
 988                                                               macAddress, ipStr, mask, actionForNode,
 989                                                               true /*isReflexsive*/);
 990                 }
 991
 992                 if (! isExternal) {
 993                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 994                             actionForNode);
 995                 }
 996                 // Enable ARP responder by default, because router interface needs to be responded always.
 997                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 998             }
 999
1000             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1001             // for the external neutron networks.
1002             //
1003             {
1004                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1005                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1006             }
1007         }
1008
1009         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1010     }
1011
1012     private void programFlowForNetworkFromExternal(final Node node,
1013                                                    final Long dpid,
1014                                                    final String destinationSegmentationId,
1015                                                    final String dstMacAddress,
1016                                                    final String destIpStr,
1017                                                    final int destMask,
1018                                                    final Action actionForNode) {
1019         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1020                 dstMacAddress, destIpStr, destMask, actionForNode);
1021     }
1022
1023     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1024                                                            final Long dpid,
1025                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1026                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1027                                                            final NeutronNetwork dstNeutronNetwork,
1028                                                            final String destinationSegmentationId,
1029                                                            final String dstMacAddress,
1030                                                            final String destIpStr,
1031                                                            final int destMask,
1032                                                            final Action actionForNode,
1033                                                            Boolean isReflexsive) {
1034         Preconditions.checkNotNull(srcNeutronRouterInterface);
1035         Preconditions.checkNotNull(dstNeutronRouterInterface);
1036
1037         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1038         if (sourceSubnetId == null) {
1039             LOG.error("Could not get provider Subnet ID from router interface {}",
1040                          srcNeutronRouterInterface.getID());
1041             return;
1042         }
1043
1044         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1045         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1046         if (sourceNetworkId == null) {
1047             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1048             return;
1049         }
1050
1051         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1052         if (sourceNetwork == null) {
1053             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1054             return;
1055         }
1056
1057         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1058             // Isolate subnets from different tenants within the same router
1059             return;
1060         }
1061         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1062         if (sourceSegmentationId == null) {
1063             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1064             return;
1065         }
1066         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1067             // Skip 'self'
1068             return;
1069         }
1070
1071         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1072                                      dstMacAddress, destIpStr, destMask, actionForNode);
1073
1074         // Flip roles src->dst; dst->src
1075         if (isReflexsive) {
1076             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1077             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1078             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1079             final String cidr2 = sourceSubnet.getCidr();
1080             final int mask2 = getMaskLenFromCidr(cidr2);
1081
1082             if (cidr2 == null || cidr2.isEmpty() ||
1083                 macAddress2 == null || macAddress2.isEmpty() ||
1084                 ipList2 == null || ipList2.isEmpty()) {
1085                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1086                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1087                 // done: go no further w/out all the info needed...
1088                 return;
1089             }
1090
1091             for (Neutron_IPs neutronIP2 : ipList2) {
1092                 final String ipStr2 = neutronIP2.getIpAddress();
1093                 if (ipStr2.isEmpty()) {
1094                     continue;
1095                 }
1096                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1097                                                           srcNeutronRouterInterface,
1098                                                           sourceNetwork, sourceSegmentationId,
1099                                                           macAddress2, ipStr2, mask2, actionForNode,
1100                                                           false /*isReflexsive*/);
1101             }
1102         }
1103     }
1104
1105     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1106                                               String destinationSegmentationId,
1107                                               String macAddress, String ipStr, int mask,
1108                                               Action actionForNode) {
1109         if (actionForNode == Action.DELETE) {
1110             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1111                          " action {}",
1112                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1113                          macAddress, ipStr, mask, actionForNode);
1114             return;
1115         }
1116         if (actionForNode == Action.ADD) {
1117             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1118                          " action {}",
1119                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1120                          macAddress, ipStr, mask, actionForNode);
1121         }
1122
1123         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1124                                                           macAddress, ipStr, mask, actionForNode);
1125     }
1126
1127     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1128                                                 String destinationSegmentationId,
1129                                                 String macAddress,
1130                                                 String address, int mask,
1131                                                 Action actionForNode) {
1132         Status status;
1133         try {
1134             InetAddress inetAddress = InetAddress.getByName(address);
1135             status = routingProvider == null ?
1136                      new Status(StatusCode.SUCCESS) :
1137                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1138                                                             macAddress, inetAddress, mask, actionForNode);
1139         } catch (UnknownHostException e) {
1140             status = new Status(StatusCode.BADREQUEST);
1141         }
1142
1143         if (status.isSuccess()) {
1144             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1145                          routingProvider == null ? "skipped" : "programmed",
1146                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1147                          actionForNode);
1148         } else {
1149             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1150                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1151                          actionForNode, status);
1152         }
1153         return status;
1154     }
1155
1156     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1157                                            String macAddress, String ipStr,
1158                                            Action action) {
1159         if (action == Action.DELETE ) {
1160             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1161                     dpid, segOrOfPort, macAddress, ipStr, action);
1162         }
1163         if (action == Action.ADD) {
1164             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1165                     dpid, segOrOfPort, macAddress, ipStr, action);
1166         }
1167
1168         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1169         return status.isSuccess();
1170     }
1171
1172     private Status programStaticArpStage2(Long dpid,
1173                                           String segOrOfPort,
1174                                           String macAddress,
1175                                           String address,
1176                                           Action action) {
1177         Status status;
1178         try {
1179             InetAddress inetAddress = InetAddress.getByName(address);
1180             status = arpProvider == null ?
1181                      new Status(StatusCode.SUCCESS) :
1182                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1183                                                        macAddress, inetAddress, action);
1184         } catch (UnknownHostException e) {
1185             status = new Status(StatusCode.BADREQUEST);
1186         }
1187
1188         if (status.isSuccess()) {
1189             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1190                          arpProvider == null ? "skipped" : "programmed",
1191                          macAddress, address, dpid, segOrOfPort, action);
1192         } else {
1193             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1194                          macAddress, address, dpid, segOrOfPort, action, status);
1195         }
1196         return status;
1197     }
1198
1199     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1200                                                   String matchAddress, String rewriteAddress,
1201                                                   Action action) {
1202         if (action == Action.DELETE ) {
1203             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1204                     " action {}",
1205                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1206         }
1207         if (action == Action.ADD ) {
1208             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1209                     " action {}",
1210                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1211         }
1212
1213         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1214                 rewriteAddress, action);
1215         return status.isSuccess();
1216     }
1217
1218     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1219                                                  String matchAddress, String rewriteAddress,
1220                                                  Action action) {
1221         Status status;
1222         try {
1223             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1224             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1225             status = inboundNatProvider == null ?
1226                     new Status(StatusCode.SUCCESS) :
1227                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1228                             inetMatchAddress, inetRewriteAddress,
1229                             action);
1230         } catch (UnknownHostException e) {
1231             status = new Status(StatusCode.BADREQUEST);
1232         }
1233
1234         if (status.isSuccess()) {
1235             final boolean isSkipped = inboundNatProvider == null;
1236             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1237                     isSkipped ? "skipped" : "programmed",
1238                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1239         } else {
1240             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1241                          " status:{}",
1242                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1243                     status);
1244         }
1245         return status;
1246     }
1247
1248     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1249                                                  Action actionForRewriteExclusion) {
1250         if (actionForRewriteExclusion == Action.DELETE ) {
1251             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1252                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1253         }
1254         if (actionForRewriteExclusion == Action.ADD) {
1255             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1256                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1257         }
1258
1259         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1260     }
1261
1262     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1263                                                    Action actionForNode) {
1264         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1265                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1266
1267         if (status.isSuccess()) {
1268             final boolean isSkipped = outboundNatProvider == null;
1269             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1270                          isSkipped ? "skipped" : "programmed",
1271                          cidr, node.getNodeId().getValue(), actionForNode);
1272         } else {
1273             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1274                          cidr, node.getNodeId().getValue(), actionForNode, status);
1275         }
1276         return status;
1277     }
1278
1279     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1280
1281         if (action == Action.DELETE) {
1282             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1283                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1284         }
1285         if (action == Action.ADD) {
1286             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1287                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1288         }
1289
1290         this.programOutboundIpRewriteStage2(fid, action);
1291     }
1292
1293     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1294         Status status;
1295         try {
1296             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1297             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1298             status = outboundNatProvider == null ?
1299                     new Status(StatusCode.SUCCESS) :
1300                     outboundNatProvider.programIpRewriteRule(
1301                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1302                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1303         } catch (UnknownHostException e) {
1304             status = new Status(StatusCode.BADREQUEST);
1305         }
1306
1307         if (status.isSuccess()) {
1308             final boolean isSkipped = outboundNatProvider == null;
1309             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1310                             " action {}",
1311                          isSkipped ? "skipped" : "programmed",
1312                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1313         } else {
1314             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1315                          " action {} status:{}",
1316                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1317         }
1318         return status;
1319     }
1320
1321     private int getMaskLenFromCidr(String cidr) {
1322         if (cidr == null) {
1323             return 0;
1324         }
1325         String[] splits = cidr.split("/");
1326         if (splits.length != 2) {
1327             return 0;
1328         }
1329
1330         int result;
1331         try {
1332             result = Integer.parseInt(splits[1].trim());
1333         } catch (NumberFormatException nfe) {
1334             result = 0;
1335         }
1336         return result;
1337     }
1338
1339     private Long getDpidForIntegrationBridge(Node node) {
1340         // Check if node is integration bridge; and only then return its dpid
1341         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1342             return southbound.getDataPathId(node);
1343         }
1344         return null;
1345     }
1346
1347     private Long getDpidForExternalBridge(Node node) {
1348         // Check if node is integration bridge; and only then return its dpid
1349         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1350             return southbound.getDataPathId(node);
1351         }
1352         return null;
1353     }
1354
1355     private Node getExternalBridgeNode(){
1356         //Pickup the first node that has external bridge (br-ex).
1357         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1358         //the external network is reachable from every br-ex
1359         // TODO: Consider other deployment scenario, and thing of better solution.
1360         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1361         for(Node node : allBridges){
1362             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1363                 return node;
1364             }
1365         }
1366         return null;
1367     }
1368
1369     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1370         if (gatewayPort.getFixedIPs() == null) {
1371             return null;
1372         }
1373         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1374             String subnetUUID = neutronIPs.getSubnetUUID();
1375             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1376             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1377                 return extSubnet;
1378             }
1379             if (extSubnet == null) {
1380                 // TODO: when subnet is created, try again.
1381                 LOG.debug("subnet {} in not found", subnetUUID);
1382              }
1383         }
1384         return null;
1385     }
1386
1387      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1388          /*
1389           *  Fix for 4277
1390           *  Remove the router cache only after deleting the neutron
1391           *  port l3 flows.
1392           */
1393          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1394
1395          if (neutronPort != null) {
1396              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1397              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1398              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1399          }
1400      }
1401
1402     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1403
1404         Preconditions.checkNotNull(node);
1405         Preconditions.checkNotNull(gatewayPort);
1406         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1407
1408         if(externalNetwork != null){
1409             if(externalNetwork.isRouterExternal()){
1410                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1411
1412                 // TODO: address IPv6 case.
1413                 if (externalSubnet != null &&
1414                     externalSubnet.getIpVersion() == 4 &&
1415                     gatewayPort.getFixedIPs() != null) {
1416                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1417                     ListenableFuture<MacAddress> gatewayMacAddress =
1418                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1419                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1420                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1421                                                              new MacAddress(gatewayPort.getMacAddress()),
1422                                                              true);
1423                     if(gatewayMacAddress != null){
1424                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1425                             @Override
1426                             public void onSuccess(MacAddress result) {
1427                                 if(result != null){
1428                                     if(!result.getValue().equals(externalRouterMac)){
1429                                         updateExternalRouterMac(result.getValue());
1430                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1431                                     }
1432                                 }else{
1433                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1434                                 }
1435                             }
1436
1437                             @Override
1438                             public void onFailure(Throwable t) {
1439                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1440                             }
1441                         }, gatewayMacResolverPool);
1442                     }
1443                 } else {
1444                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1445                 }
1446             }
1447         }else{
1448             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1449         }
1450     }
1451
1452     /**
1453      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1454      *
1455      * @param ofPort the OF port number
1456      * @return the string with encoded OF port (example format "OFPort|999")
1457      */
1458     public static String encodeExcplicitOFPort(Long ofPort) {
1459         return "OFPort|" + ofPort.toString();
1460     }
1461
1462     @Override
1463     public void setDependencies(ServiceReference serviceReference) {
1464         tenantNetworkManager =
1465                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1466         configurationService =
1467                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1468         arpProvider =
1469                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1470         inboundNatProvider =
1471                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1472         outboundNatProvider =
1473                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1474         routingProvider =
1475                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1476         l3ForwardingProvider =
1477                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1478         nodeCacheManager =
1479                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1480         southbound =
1481                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1482         gatewayMacResolver =
1483                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1484         securityServicesManager =
1485                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1486         initL3AdapterMembers();
1487     }
1488
1489     @Override
1490     public void setDependencies(Object impl) {
1491         if (impl instanceof INeutronNetworkCRUD) {
1492             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1493         } else if (impl instanceof INeutronPortCRUD) {
1494             neutronPortCache = (INeutronPortCRUD)impl;
1495         } else if (impl instanceof INeutronSubnetCRUD) {
1496             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1497         } else if (impl instanceof INeutronFloatingIPCRUD) {
1498             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1499         } else if (impl instanceof ArpProvider) {
1500             arpProvider = (ArpProvider)impl;
1501         } else if (impl instanceof InboundNatProvider) {
1502             inboundNatProvider = (InboundNatProvider)impl;
1503         } else if (impl instanceof OutboundNatProvider) {
1504             outboundNatProvider = (OutboundNatProvider)impl;
1505         } else if (impl instanceof RoutingProvider) {
1506             routingProvider = (RoutingProvider)impl;
1507         } else if (impl instanceof L3ForwardingProvider) {
1508             l3ForwardingProvider = (L3ForwardingProvider)impl;
1509         }else if (impl instanceof GatewayMacResolver) {
1510             gatewayMacResolver = (GatewayMacResolver)impl;
1511         }
1512         populateL3ForwardingCaches();
1513     }
1514 }