openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import java.net.InetAddress;
  12 import java.net.UnknownHostException;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.ExecutorService;
  20 import java.util.concurrent.Executors;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  49 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  50 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  51 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  52 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  53 import org.opendaylight.ovsdb.utils.mdsal.utils.NeutronModelsDataStoreHelper;
  54 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  63 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  64 import org.osgi.framework.ServiceReference;
  65 import org.slf4j.Logger;
  66 import org.slf4j.LoggerFactory;
  67
  68 import com.google.common.base.Preconditions;
  69 import com.google.common.util.concurrent.FutureCallback;
  70 import com.google.common.util.concurrent.Futures;
  71 import com.google.common.util.concurrent.ListenableFuture;
  72
  73 /**
  74  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  75  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  76  * as well as the multi-tenant router forwarding provider.
  77  */
  78 public class NeutronL3Adapter implements ConfigInterface {
  79     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  80
  81     // The implementation for each of these services is resolved by the OSGi Service Manager
  82     private volatile ConfigurationService configurationService;
  83     private volatile TenantNetworkManager tenantNetworkManager;
  84     private volatile NodeCacheManager nodeCacheManager;
  85     private volatile INeutronNetworkCRUD neutronNetworkCache;
  86     private volatile INeutronSubnetCRUD neutronSubnetCache;
  87     private volatile INeutronPortCRUD neutronPortCache;
  88     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  89     private volatile L3ForwardingProvider l3ForwardingProvider;
  90     private volatile InboundNatProvider inboundNatProvider;
  91     private volatile OutboundNatProvider outboundNatProvider;
  92     private volatile ArpProvider arpProvider;
  93     private volatile RoutingProvider routingProvider;
  94     private volatile GatewayMacResolver gatewayMacResolver;
  95     private volatile SecurityServicesManager securityServicesManager;
  96
  97     private class FloatIpData {
  98         // br-int of node where floating ip is associated with tenant port
  99         private final Long dpid;
 100         // patch port in br-int used to reach br-ex
 101         private final Long ofPort;
 102         // segmentation id of the net where fixed ip is instantiated
 103         private final String segId;
 104         // mac address assigned to neutron port of floating ip
 105         private final String macAddress;
 106         private final String floatingIpAddress;
 107         // ip address given to tenant vm
 108         private final String fixedIpAddress;
 109         private final String neutronRouterMac;
 110
 111         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 112                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 113             this.dpid = dpid;
 114             this.ofPort = ofPort;
 115             this.segId = segId;
 116             this.macAddress = macAddress;
 117             this.floatingIpAddress = floatingIpAddress;
 118             this.fixedIpAddress = fixedIpAddress;
 119             this.neutronRouterMac = neutronRouterMac;
 120         }
 121     }
 122
 123     private Map<String, String> networkIdToRouterMacCache;
 124     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 125     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 126
 127     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 128     private Map<String, FloatIpData> floatIpDataMapCache;
 129
 130     private String externalRouterMac;
 131     private Boolean enabled = false;
 132     private Boolean flgDistributedARPEnabled = true;
 133     private Boolean isCachePopulationDone = false;
 134     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 135
 136     private Southbound southbound;
 137     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 138
 139     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 140     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 141     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 142     private static final String OWNER_FLOATING_IP = "network:floatingip";
 143     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 144
 145     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 146         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 147         this.neutronModelsDataStoreHelper = neutronHelper;
 148     }
 149
 150     private void initL3AdapterMembers() {
 151         Preconditions.checkNotNull(configurationService);
 152
 153         if (configurationService.isL3ForwardingEnabled()) {
 154             this.networkIdToRouterMacCache = new HashMap<>();
 155             this.networkIdToRouterIpListCache = new HashMap<>();
 156             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 157             this.neutronPortToDpIdCache = new HashMap<>();
 158             this.floatIpDataMapCache = new HashMap<>();
 159
 160             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 161             if (this.externalRouterMac == null) {
 162                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 163             }
 164
 165             this.enabled = true;
 166             LOG.info("OVSDB L3 forwarding is enabled");
 167             if (configurationService.isDistributedArpDisabled()) {
 168                 this.flgDistributedARPEnabled = false;
 169                 LOG.info("Distributed ARP responder is disabled");
 170             } else {
 171                 LOG.debug("Distributed ARP responder is enabled");
 172             }
 173         } else {
 174             LOG.debug("OVSDB L3 forwarding is disabled");
 175         }
 176     }
 177
 178
 179     private void populateL3ForwardingCaches() {
 180         if (!this.enabled) {
 181             return;
 182         }
 183         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 184                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 185             return;
 186         }
 187         this.isCachePopulationDone = true;
 188         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 189         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 190         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 191         if(routers != null && routers.getRouter() != null && ports != null) {
 192             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 193             for( Router router : routers.getRouter()) {
 194                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 195                 if(!ports.getPort().isEmpty()) {
 196                     for( Port port : ports.getPort()) {
 197                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 198                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 199                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 200                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 201                                     , port.getMacAddress());
 202
 203                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 204                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 205                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 206                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 207                         }
 208                     }
 209                 }else {
 210                     LOG.warn("L3 Cache Population :Did not find any port information " +
 211                             "in config Data Store for router {}",router);
 212                 }
 213             }
 214         }
 215         LOG.debug("NetVirt L3 caches population is done");
 216     }
 217
 218     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 219         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 220             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 221             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 222             for(Node bridge : bridges) {
 223                 List<OvsdbTerminationPointAugmentation> interfaces
 224                         = southbound.extractTerminationPointAugmentations(bridge);
 225                 if(interfaces != null && !interfaces.isEmpty()) {
 226                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 227                             interfaces.size(), bridge.getNodeId());
 228                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 229                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 230                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 231                             Long dpId = getDpidForIntegrationBridge(bridge);
 232                             Uuid interfaceUuid = intf.getInterfaceUuid();
 233                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 234                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 235                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 236                             break;
 237                         }
 238                     }
 239                 }
 240             }
 241         }
 242         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 243     }
 244
 245     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 246         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 247         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 248         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 249             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 250                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 251                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 252                             "Fetching from data store.",neutronFloatingIP.getID());
 253                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 254                 }
 255             }
 256         }
 257         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 258         return floatIpDataMapCache.values();
 259     }
 260     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 261         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 262                 "floating id {} ",neutronFloatingId);
 263         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 264             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 265             if (neutronFloatingIP == null) {
 266                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 267                 return null;
 268             }
 269             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 270             NeutronPort neutronPortForFloatIp = null;
 271             for (NeutronPort neutronPort : neutronPorts) {
 272                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 273                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 274                     neutronPortForFloatIp = neutronPort;
 275                     break;
 276                 }
 277             }
 278
 279             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 280             if(neutronTenantPortUuid == null) {
 281                 return null;
 282             }
 283             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 284             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 285             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 286             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 287
 288             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 289             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 290                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 291             String providerSegmentationId = tenantNeutronNetwork != null ?
 292                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 293             String neutronRouterMac = tenantNeutronNetwork != null ?
 294                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 295
 296             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 297                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 298                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 299                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 300                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 301                                 "seg {} mac {} rtrMac {}",
 302                         fixedIpAddress,
 303                         floatingIpAddress,
 304                         neutronPortForFloatIp,
 305                         neutronTenantPortUuid,
 306                         providerSegmentationId,
 307                         floatingIpMac,
 308                         neutronRouterMac);
 309
 310                 return null;
 311             }
 312
 313             // get ofport for patch port in br-int
 314             final Long dpId = nodeIfPair.getLeft();
 315             final Long ofPort = findOFPortForExtPatch(dpId);
 316             if (ofPort == null) {
 317                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 318                                 "to connect floating ip to external bridge. dpid {}",
 319                         dpId);
 320                 return null;
 321             }
 322
 323             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 324                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 325             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 326
 327         }
 328         return floatIpDataMapCache.get(neutronFloatingId);
 329     }
 330     /**
 331      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 332      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 333      * subscribe so that the mac address associated to the gateway ip address is available.
 334      *
 335      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 336      */
 337     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 338         Preconditions.checkNotNull(externalRouterMacUpdate);
 339
 340         flushExistingIpRewrite();
 341         this.externalRouterMac = externalRouterMacUpdate;
 342         rebuildExistingIpRewrite();
 343     }
 344
 345     /**
 346      * Process the event.
 347      *
 348      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 349      * @param subnet An instance of NeutronSubnet object.
 350      */
 351     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 352         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 353     }
 354
 355     /**
 356      * Process the port event as a router interface event.
 357      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 358      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 359      *
 360      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 361      * @param neutronPort An instance of NeutronPort object.
 362      */
 363     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 364         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 365
 366         this.processSecurityGroupUpdate(neutronPort);
 367         if (!this.enabled) {
 368             return;
 369         }
 370
 371         final boolean isDelete = action == Action.DELETE;
 372
 373         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 374             if(!isDelete){
 375                 Node externalBridgeNode = getExternalBridgeNode();
 376                 if(externalBridgeNode != null){
 377                     LOG.info("Port {} is network router gateway interface, "
 378                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 379                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 380                 }else{
 381                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 382                 }
 383             }else{
 384                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 385
 386                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 387                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 388                     // TODO support IPv6
 389                     if (externalSubnet != null &&
 390                             externalSubnet.getIpVersion() == 4) {
 391                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 392                     }
 393                 }
 394             }
 395         }
 396
 397         // Treat the port event as a router interface event if the port belongs to router. This is a
 398         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 399         //
 400         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 401             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 402
 403             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 404                 NeutronRouter_Interface neutronRouterInterface =
 405                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 406                 // id of router interface to be same as subnet
 407                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 408                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 409
 410                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 411             }
 412         } else {
 413             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 414             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 415             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 416             // there.
 417             //
 418             if (!isDelete && neutronPort.getFixedIPs() != null) {
 419                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 420                     NeutronRouter_Interface neutronRouterInterface =
 421                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 422                     if (neutronRouterInterface != null) {
 423                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 424                     }
 425                 }
 426             }
 427             this.updateL3ForNeutronPort(neutronPort, isDelete);
 428         }
 429     }
 430
 431     /**
 432      * Process the event.
 433      *
 434      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 435      * @param neutronRouter An instance of NeutronRouter object.
 436      */
 437     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 438         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 439     }
 440
 441     /**
 442      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 443      * delete the ports on the same subnet.
 444      *
 445      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 446      * @param neutronRouter An instance of NeutronRouter object.
 447      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 448      */
 449     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 450                                                   final NeutronRouter_Interface neutronRouterInterface,
 451                                                   Action action) {
 452         LOG.debug("Router interface {} got event {}. Subnet {}",
 453                      neutronRouterInterface.getPortUUID(),
 454                      action,
 455                      neutronRouterInterface.getSubnetUUID());
 456         if (!this.enabled) {
 457             return;
 458         }
 459
 460         final boolean isDelete = action == Action.DELETE;
 461
 462         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 463
 464         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 465         // see if they are affected by l3
 466         //
 467         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 468             boolean currPortShouldBeDeleted = false;
 469             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 470             if (isDelete) {
 471                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 472                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 473                         currPortShouldBeDeleted = true;
 474                         break;
 475                     }
 476                 }
 477             }
 478             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 479         }
 480
 481         if (isDelete) {
 482             /*
 483              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 484              */
 485             this.cleanupRouterCache(neutronRouterInterface);
 486         }
 487     }
 488
 489     /**
 490      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 491      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 492      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 493      * way around) on OpenFlow Table 100.
 494      *
 495      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 496      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 497      */
 498     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 499                                              Action actionIn) {
 500         Preconditions.checkNotNull(neutronFloatingIP);
 501
 502         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 503                 neutronFloatingIP.getFixedIPAddress(),
 504                 neutronFloatingIP.getFloatingIPAddress(),
 505                 neutronFloatingIP.getFloatingNetworkUUID());
 506         if (!this.enabled) {
 507             return;
 508         }
 509
 510         Action action;
 511
 512         // Consider action to be delete if getFixedIPAddress is null
 513         //
 514         if (neutronFloatingIP.getFixedIPAddress() == null) {
 515             action = Action.DELETE;
 516         } else {
 517             action = actionIn;
 518         }
 519
 520         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 521
 522         if (action != Action.DELETE) {
 523             // must be first, as it updates floatIpDataMapCache
 524             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 525
 526             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 527             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 528         } else {
 529             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 530             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 531
 532             // must be last, as it updates floatIpDataMapCache
 533             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 534         }
 535     }
 536
 537     /**
 538      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 539      * ip, otherwise for newer one.
 540      */
 541     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 542         Preconditions.checkNotNull(neutronFloatingIP);
 543
 544         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 545         if (fid == null) {
 546             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 547                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 548             return;
 549         }
 550         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 551                                       action);
 552     }
 553
 554     /**
 555      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 556      * ip, otherwise for newer one.
 557      */
 558     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 559         Preconditions.checkNotNull(neutronFloatingIP);
 560
 561         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 562         if (fid == null) {
 563             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 564                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 565             return;
 566         }
 567         programOutboundIpRewriteStage1(fid, action);
 568     }
 569
 570     private void flushExistingIpRewrite() {
 571         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 572             programOutboundIpRewriteStage1(fid, Action.DELETE);
 573         }
 574     }
 575
 576     private void rebuildExistingIpRewrite() {
 577         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 578             programOutboundIpRewriteStage1(fid, Action.ADD);
 579         }
 580     }
 581
 582     /**
 583      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 584      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 585      * requests.
 586      */
 587     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 588         Preconditions.checkNotNull(neutronFloatingIP);
 589         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 590         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 591
 592         // find bridge Node where floating ip is configured by looking up cache for its port
 593         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 594         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 595         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 596         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 597         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 598         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 599
 600         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 601         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 602                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 603         final String providerSegmentationId = tenantNeutronNetwork != null ?
 604                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 605         final String neutronRouterMac = tenantNeutronNetwork != null ?
 606                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 607
 608         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 609                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 610                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 611                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 612             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 613                     fixedIpAddress,
 614                     floatingIpAddress,
 615                     neutronPortForFloatIp,
 616                     neutronTenantPortUuid,
 617                     providerSegmentationId,
 618                     floatingIpMac,
 619                     neutronRouterMac);
 620             return;
 621         }
 622
 623         // get ofport for patch port in br-int
 624         final Long dpId = nodeIfPair.getLeft();
 625         final Long ofPort = findOFPortForExtPatch(dpId);
 626         if (ofPort == null) {
 627             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 628                     dpId);
 629             return;
 630         }
 631
 632         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 633         //
 634         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 635                 Action.ADD)) {
 636             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 637                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 638             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 639             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 640                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 641                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 642         }
 643     }
 644
 645     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 646         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 647         if (floatIpData == null) {
 648             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 649             return;
 650         }
 651
 652         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 653                 floatIpData.floatingIpAddress, Action.DELETE)) {
 654             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 655             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 656                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 657         }
 658     }
 659
 660     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 661         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 662             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 663                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 664                 return neutronPort;
 665             }
 666         }
 667         return null;
 668     }
 669
 670     private Long findOFPortForExtPatch(Long dpId) {
 671         final String brInt = configurationService.getIntegrationBridgeName();
 672         final String brExt = configurationService.getExternalBridgeName();
 673         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 674
 675         Preconditions.checkNotNull(dpId);
 676         Preconditions.checkNotNull(portNameInt);
 677
 678         final long dpidPrimitive = dpId;
 679         for (Node node : nodeCacheManager.getBridgeNodes()) {
 680             if (dpidPrimitive == southbound.getDataPathId(node)) {
 681                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 682                         southbound.getTerminationPointOfBridge(node, portNameInt);
 683                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 684             }
 685         }
 686         return null;
 687     }
 688
 689     /**
 690      * Process the event.
 691      *
 692      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 693      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 694      */
 695     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 696         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 697     }
 698
 699     //
 700     // Callbacks from OVSDB's southbound handler
 701     //
 702     /**
 703      * Process the event.
 704      *
 705      * @param bridgeNode An instance of Node object.
 706      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 707      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 708      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 709      * object.
 710      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 711      */
 712     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 713                                      final NeutronNetwork neutronNetwork, Action action) {
 714         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 715                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 716         if (!this.enabled) {
 717             return;
 718         }
 719
 720         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 721         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 722         final Uuid interfaceUuid = intf.getInterfaceUuid();
 723
 724         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 725                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 726
 727         if (neutronPort != null) {
 728             final String neutronPortUuid = neutronPort.getPortUUID();
 729
 730             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 731                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 732             }
 733
 734             handleNeutronPortEvent(neutronPort, action);
 735         }
 736
 737         if (action == Action.DELETE && interfaceUuid != null) {
 738             handleInterfaceEventDelete(intf, dpId);
 739         }
 740     }
 741
 742     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 743         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 744         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 745                 neutronPortUuid, dpId, interfaceUuid.getValue());
 746     }
 747
 748     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 749         // Remove entry from neutronPortToDpIdCache based on interface uuid
 750         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 751             final String currPortUuid = entry.getKey();
 752             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 753                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 754                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 755                 neutronPortToDpIdCache.remove(currPortUuid);
 756                 break;
 757             }
 758         }
 759     }
 760
 761     //
 762     // Internal helpers
 763     //
 764     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 765
 766         final String networkUUID = neutronPort.getNetworkUUID();
 767         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 768
 769         // If there is no router interface handling the networkUUID, we are done
 770         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 771             return;
 772         }
 773
 774         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 775         // router interface are handled via handleNeutronRouterInterfaceEvent.
 776         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 777             return;
 778         }
 779
 780         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 781         final String providerSegmentationId = neutronNetwork != null ?
 782                                               neutronNetwork.getProviderSegmentationID() : null;
 783         final String tenantMac = neutronPort.getMacAddress();
 784
 785         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 786             tenantMac == null || tenantMac.isEmpty()) {
 787             // done: go no further w/out all the info needed...
 788             return;
 789         }
 790
 791         final Action action = isDelete ? Action.DELETE : Action.ADD;
 792         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 793         if (nodes.isEmpty()) {
 794             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 795         }
 796         for (Node node : nodes) {
 797             final Long dpid = getDpidForIntegrationBridge(node);
 798             if (dpid == null) {
 799                 continue;
 800             }
 801
 802             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 803                 final String tenantIpStr = neutronIP.getIpAddress();
 804                 if (tenantIpStr.isEmpty()) {
 805                     continue;
 806                 }
 807
 808                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 809                 // still needed when routing to subnets non-local to node (bug 2076).
 810                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 811
 812                 // Configure distributed ARP responder
 813                 if (flgDistributedARPEnabled) {
 814                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 815                 }
 816             }
 817         }
 818     }
 819
 820     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 821         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 822         /**
 823          * Get updated data and original data for the the changed. Identify the security groups that got
 824          * added and removed and call the appropriate providers for updating the flows.
 825          */
 826         try {
 827             NeutronPort originalPort = neutronPort.getOriginalPort();
 828             if (null == originalPort) {
 829                 LOG.debug("processSecurityGroupUpdate: originalport is empty");
 830                 return;
 831             }
 832             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 833                                                                             neutronPort.getOriginalPort());
 834             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 835                                                                               neutronPort);
 836
 837             if (null != addedGroup && !addedGroup.isEmpty()) {
 838                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 839             }
 840             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 841                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 842             }
 843
 844         } catch (Exception e) {
 845             LOG.error("Exception in processSecurityGroupUpdate", e);
 846         }
 847     }
 848
 849     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 850         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 851         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 852         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 853         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 854             NeutronSecurityGroup securityGroup1 = iterator.next();
 855             for (NeutronSecurityGroup securityGroup2 :list2) {
 856                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 857                     iterator.remove();
 858                 }
 859             }
 860         }
 861         return list1;
 862     }
 863
 864     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 865                                            String macAddress, String ipStr,
 866                                            Action actionForNode) {
 867         if (actionForNode == Action.DELETE) {
 868             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 869                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 870         }
 871         if (actionForNode == Action.ADD) {
 872             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 873                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 874         }
 875
 876         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 877                                                        macAddress, ipStr, actionForNode);
 878     }
 879
 880     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 881                                              String macAddress,
 882                                              String address,
 883                                              Action actionForNode) {
 884         Status status;
 885         try {
 886             InetAddress inetAddress = InetAddress.getByName(address);
 887             status = l3ForwardingProvider == null ?
 888                      new Status(StatusCode.SUCCESS) :
 889                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 890                                                                       inetAddress, macAddress, actionForNode);
 891         } catch (UnknownHostException e) {
 892             status = new Status(StatusCode.BADREQUEST);
 893         }
 894
 895         if (status.isSuccess()) {
 896             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 897                          l3ForwardingProvider == null ? "skipped" : "programmed",
 898                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 899         } else {
 900             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 901                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 902         }
 903         return status;
 904     }
 905
 906     // --
 907
 908     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 909                                                        Boolean isDelete) {
 910         Preconditions.checkNotNull(destNeutronRouterInterface);
 911
 912         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 913         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 914         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 915         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 916         final NeutronNetwork neutronNetwork = subnet != null ?
 917                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 918         final String destinationSegmentationId = neutronNetwork != null ?
 919                                                  neutronNetwork.getProviderSegmentationID() : null;
 920         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 921         final String cidr = subnet != null ? subnet.getCidr() : null;
 922         final int mask = getMaskLenFromCidr(cidr);
 923
 924         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 925                      destNeutronRouterInterface, isDelete);
 926
 927         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 928         // the local cache
 929         if (neutronNetwork != null) {
 930             if (macAddress == null || macAddress.isEmpty()) {
 931                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 932             }
 933             if (ipList == null || ipList.isEmpty()) {
 934                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 935             }
 936         }
 937
 938         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 939             cidr == null || cidr.isEmpty() ||
 940             macAddress == null || macAddress.isEmpty() ||
 941             ipList == null || ipList.isEmpty()) {
 942             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 943                          destinationSegmentationId, cidr, macAddress, ipList);
 944             // done: go no further w/out all the info needed...
 945             return;
 946         }
 947
 948         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 949
 950         // Keep cache for finding router's mac from network uuid -- add
 951         //
 952         if (! isDelete) {
 953             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 954             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 955             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 956         }
 957
 958         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 959         if (nodes.isEmpty()) {
 960             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 961         }
 962         for (Node node : nodes) {
 963             final Long dpid = getDpidForIntegrationBridge(node);
 964             if (dpid == null) {
 965                 continue;
 966             }
 967
 968             for (Neutron_IPs neutronIP : ipList) {
 969                 final String ipStr = neutronIP.getIpAddress();
 970                 if (ipStr.isEmpty()) {
 971                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 972                             node.getNodeId().getValue(), ipStr);
 973                     continue;
 974                 }
 975
 976                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 977                 //
 978                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 979                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 980                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 981                                                               neutronNetwork, destinationSegmentationId,
 982                                                               macAddress, ipStr, mask, actionForNode,
 983                                                               true /*isReflexsive*/);
 984                 }
 985
 986                 if (! isExternal) {
 987                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 988                             actionForNode);
 989                 }
 990                 // Enable ARP responder by default, because router interface needs to be responded always.
 991                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 992             }
 993
 994             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 995             // for the external neutron networks.
 996             //
 997             {
 998                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 999                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1000             }
1001         }
1002
1003         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1004     }
1005
1006     private void programFlowForNetworkFromExternal(final Node node,
1007                                                    final Long dpid,
1008                                                    final String destinationSegmentationId,
1009                                                    final String dstMacAddress,
1010                                                    final String destIpStr,
1011                                                    final int destMask,
1012                                                    final Action actionForNode) {
1013         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1014                 dstMacAddress, destIpStr, destMask, actionForNode);
1015     }
1016
1017     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1018                                                            final Long dpid,
1019                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1020                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1021                                                            final NeutronNetwork dstNeutronNetwork,
1022                                                            final String destinationSegmentationId,
1023                                                            final String dstMacAddress,
1024                                                            final String destIpStr,
1025                                                            final int destMask,
1026                                                            final Action actionForNode,
1027                                                            Boolean isReflexsive) {
1028         Preconditions.checkNotNull(srcNeutronRouterInterface);
1029         Preconditions.checkNotNull(dstNeutronRouterInterface);
1030
1031         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1032         if (sourceSubnetId == null) {
1033             LOG.error("Could not get provider Subnet ID from router interface {}",
1034                          srcNeutronRouterInterface.getID());
1035             return;
1036         }
1037
1038         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1039         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1040         if (sourceNetworkId == null) {
1041             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1042             return;
1043         }
1044
1045         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1046         if (sourceNetwork == null) {
1047             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1048             return;
1049         }
1050
1051         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1052             // Isolate subnets from different tenants within the same router
1053             return;
1054         }
1055         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1056         if (sourceSegmentationId == null) {
1057             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1058             return;
1059         }
1060         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1061             // Skip 'self'
1062             return;
1063         }
1064
1065         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1066                                      dstMacAddress, destIpStr, destMask, actionForNode);
1067
1068         // Flip roles src->dst; dst->src
1069         if (isReflexsive) {
1070             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1071             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1072             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1073             final String cidr2 = sourceSubnet.getCidr();
1074             final int mask2 = getMaskLenFromCidr(cidr2);
1075
1076             if (cidr2 == null || cidr2.isEmpty() ||
1077                 macAddress2 == null || macAddress2.isEmpty() ||
1078                 ipList2 == null || ipList2.isEmpty()) {
1079                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1080                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1081                 // done: go no further w/out all the info needed...
1082                 return;
1083             }
1084
1085             for (Neutron_IPs neutronIP2 : ipList2) {
1086                 final String ipStr2 = neutronIP2.getIpAddress();
1087                 if (ipStr2.isEmpty()) {
1088                     continue;
1089                 }
1090                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1091                                                           srcNeutronRouterInterface,
1092                                                           sourceNetwork, sourceSegmentationId,
1093                                                           macAddress2, ipStr2, mask2, actionForNode,
1094                                                           false /*isReflexsive*/);
1095             }
1096         }
1097     }
1098
1099     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1100                                               String destinationSegmentationId,
1101                                               String macAddress, String ipStr, int mask,
1102                                               Action actionForNode) {
1103         if (actionForNode == Action.DELETE) {
1104             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1105                          " action {}",
1106                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1107                          macAddress, ipStr, mask, actionForNode);
1108             return;
1109         }
1110         if (actionForNode == Action.ADD) {
1111             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1112                          " action {}",
1113                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1114                          macAddress, ipStr, mask, actionForNode);
1115         }
1116
1117         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1118                                                           macAddress, ipStr, mask, actionForNode);
1119     }
1120
1121     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1122                                                 String destinationSegmentationId,
1123                                                 String macAddress,
1124                                                 String address, int mask,
1125                                                 Action actionForNode) {
1126         Status status;
1127         try {
1128             InetAddress inetAddress = InetAddress.getByName(address);
1129             status = routingProvider == null ?
1130                      new Status(StatusCode.SUCCESS) :
1131                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1132                                                             macAddress, inetAddress, mask, actionForNode);
1133         } catch (UnknownHostException e) {
1134             status = new Status(StatusCode.BADREQUEST);
1135         }
1136
1137         if (status.isSuccess()) {
1138             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1139                          routingProvider == null ? "skipped" : "programmed",
1140                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1141                          actionForNode);
1142         } else {
1143             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1144                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1145                          actionForNode, status);
1146         }
1147         return status;
1148     }
1149
1150     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1151                                            String macAddress, String ipStr,
1152                                            Action action) {
1153         if (action == Action.DELETE ) {
1154             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1155                     dpid, segOrOfPort, macAddress, ipStr, action);
1156         }
1157         if (action == Action.ADD) {
1158             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1159                     dpid, segOrOfPort, macAddress, ipStr, action);
1160         }
1161
1162         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1163         return status.isSuccess();
1164     }
1165
1166     private Status programStaticArpStage2(Long dpid,
1167                                           String segOrOfPort,
1168                                           String macAddress,
1169                                           String address,
1170                                           Action action) {
1171         Status status;
1172         try {
1173             InetAddress inetAddress = InetAddress.getByName(address);
1174             status = arpProvider == null ?
1175                      new Status(StatusCode.SUCCESS) :
1176                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1177                                                        macAddress, inetAddress, action);
1178         } catch (UnknownHostException e) {
1179             status = new Status(StatusCode.BADREQUEST);
1180         }
1181
1182         if (status.isSuccess()) {
1183             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1184                          arpProvider == null ? "skipped" : "programmed",
1185                          macAddress, address, dpid, segOrOfPort, action);
1186         } else {
1187             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1188                          macAddress, address, dpid, segOrOfPort, action, status);
1189         }
1190         return status;
1191     }
1192
1193     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1194                                                   String matchAddress, String rewriteAddress,
1195                                                   Action action) {
1196         if (action == Action.DELETE ) {
1197             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1198                     " action {}",
1199                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1200         }
1201         if (action == Action.ADD ) {
1202             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1203                     " action {}",
1204                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1205         }
1206
1207         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1208                 rewriteAddress, action);
1209         return status.isSuccess();
1210     }
1211
1212     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1213                                                  String matchAddress, String rewriteAddress,
1214                                                  Action action) {
1215         Status status;
1216         try {
1217             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1218             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1219             status = inboundNatProvider == null ?
1220                     new Status(StatusCode.SUCCESS) :
1221                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1222                             inetMatchAddress, inetRewriteAddress,
1223                             action);
1224         } catch (UnknownHostException e) {
1225             status = new Status(StatusCode.BADREQUEST);
1226         }
1227
1228         if (status.isSuccess()) {
1229             final boolean isSkipped = inboundNatProvider == null;
1230             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1231                     isSkipped ? "skipped" : "programmed",
1232                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1233         } else {
1234             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1235                          " status:{}",
1236                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1237                     status);
1238         }
1239         return status;
1240     }
1241
1242     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1243                                                  Action actionForRewriteExclusion) {
1244         if (actionForRewriteExclusion == Action.DELETE ) {
1245             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1246                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1247         }
1248         if (actionForRewriteExclusion == Action.ADD) {
1249             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1250                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1251         }
1252
1253         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1254     }
1255
1256     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1257                                                    Action actionForNode) {
1258         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1259                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1260
1261         if (status.isSuccess()) {
1262             final boolean isSkipped = outboundNatProvider == null;
1263             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1264                          isSkipped ? "skipped" : "programmed",
1265                          cidr, node.getNodeId().getValue(), actionForNode);
1266         } else {
1267             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1268                          cidr, node.getNodeId().getValue(), actionForNode, status);
1269         }
1270         return status;
1271     }
1272
1273     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1274
1275         if (action == Action.DELETE) {
1276             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1277                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1278         }
1279         if (action == Action.ADD) {
1280             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1281                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1282         }
1283
1284         this.programOutboundIpRewriteStage2(fid, action);
1285     }
1286
1287     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1288         Status status;
1289         try {
1290             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1291             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1292             status = outboundNatProvider == null ?
1293                     new Status(StatusCode.SUCCESS) :
1294                     outboundNatProvider.programIpRewriteRule(
1295                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1296                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1297         } catch (UnknownHostException e) {
1298             status = new Status(StatusCode.BADREQUEST);
1299         }
1300
1301         if (status.isSuccess()) {
1302             final boolean isSkipped = outboundNatProvider == null;
1303             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1304                             " action {}",
1305                          isSkipped ? "skipped" : "programmed",
1306                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1307         } else {
1308             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1309                          " action {} status:{}",
1310                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1311         }
1312         return status;
1313     }
1314
1315     private int getMaskLenFromCidr(String cidr) {
1316         if (cidr == null) {
1317             return 0;
1318         }
1319         String[] splits = cidr.split("/");
1320         if (splits.length != 2) {
1321             return 0;
1322         }
1323
1324         int result;
1325         try {
1326             result = Integer.parseInt(splits[1].trim());
1327         } catch (NumberFormatException nfe) {
1328             result = 0;
1329         }
1330         return result;
1331     }
1332
1333     private Long getDpidForIntegrationBridge(Node node) {
1334         // Check if node is integration bridge; and only then return its dpid
1335         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1336             return southbound.getDataPathId(node);
1337         }
1338         return null;
1339     }
1340
1341     private Long getDpidForExternalBridge(Node node) {
1342         // Check if node is integration bridge; and only then return its dpid
1343         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1344             return southbound.getDataPathId(node);
1345         }
1346         return null;
1347     }
1348
1349     private Node getExternalBridgeNode(){
1350         //Pickup the first node that has external bridge (br-ex).
1351         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1352         //the external network is reachable from every br-ex
1353         // TODO: Consider other deployment scenario, and thing of better solution.
1354         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1355         for(Node node : allBridges){
1356             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1357                 return node;
1358             }
1359         }
1360         return null;
1361     }
1362
1363     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1364         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1365             String subnetUUID = neutronIPs.getSubnetUUID();
1366             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1367             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1368                 return extSubnet;
1369             }
1370             if (extSubnet == null) {
1371                 // TODO: when subnet is created, try again.
1372                 LOG.debug("subnet {} in not found", subnetUUID);
1373              }
1374         }
1375         return null;
1376     }
1377
1378      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1379          /*
1380           *  Fix for 4277
1381           *  Remove the router cache only after deleting the neutron
1382           *  port l3 flows.
1383           */
1384          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1385
1386          if (neutronPort != null) {
1387              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1388              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1389              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1390          }
1391      }
1392
1393     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1394
1395         Preconditions.checkNotNull(node);
1396         Preconditions.checkNotNull(gatewayPort);
1397         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1398
1399         if(externalNetwork != null){
1400             if(externalNetwork.isRouterExternal()){
1401                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1402
1403                 // TODO: address IPv6 case.
1404                 if (externalSubnet != null &&
1405                     externalSubnet.getIpVersion() == 4) {
1406                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1407                     ListenableFuture<MacAddress> gatewayMacAddress =
1408                         gatewayMacResolver.resolveMacAddress(getDpidForExternalBridge(node),
1409                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1410                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1411                                                              new MacAddress(gatewayPort.getMacAddress()),
1412                                                              true);
1413                     if(gatewayMacAddress != null){
1414                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1415                             @Override
1416                             public void onSuccess(MacAddress result) {
1417                                 if(result != null){
1418                                     if(!result.getValue().equals(externalRouterMac)){
1419                                         updateExternalRouterMac(result.getValue());
1420                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1421                                     }
1422                                 }else{
1423                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1424                                 }
1425                             }
1426
1427                             @Override
1428                             public void onFailure(Throwable t) {
1429                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1430                             }
1431                         }, gatewayMacResolverPool);
1432                     }
1433                 } else {
1434                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1435                 }
1436             }
1437         }else{
1438             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1439         }
1440     }
1441
1442     /**
1443      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1444      *
1445      * @param ofPort the OF port number
1446      * @return the string with encoded OF port (example format "OFPort|999")
1447      */
1448     public static String encodeExcplicitOFPort(Long ofPort) {
1449         return "OFPort|" + ofPort.toString();
1450     }
1451
1452     @Override
1453     public void setDependencies(ServiceReference serviceReference) {
1454         tenantNetworkManager =
1455                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1456         configurationService =
1457                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1458         arpProvider =
1459                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1460         inboundNatProvider =
1461                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1462         outboundNatProvider =
1463                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1464         routingProvider =
1465                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1466         l3ForwardingProvider =
1467                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1468         nodeCacheManager =
1469                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1470         southbound =
1471                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1472         gatewayMacResolver =
1473                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1474         securityServicesManager =
1475                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1476         initL3AdapterMembers();
1477     }
1478
1479     @Override
1480     public void setDependencies(Object impl) {
1481         if (impl instanceof INeutronNetworkCRUD) {
1482             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1483         } else if (impl instanceof INeutronPortCRUD) {
1484             neutronPortCache = (INeutronPortCRUD)impl;
1485         } else if (impl instanceof INeutronSubnetCRUD) {
1486             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1487         } else if (impl instanceof INeutronFloatingIPCRUD) {
1488             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1489         } else if (impl instanceof ArpProvider) {
1490             arpProvider = (ArpProvider)impl;
1491         } else if (impl instanceof InboundNatProvider) {
1492             inboundNatProvider = (InboundNatProvider)impl;
1493         } else if (impl instanceof OutboundNatProvider) {
1494             outboundNatProvider = (OutboundNatProvider)impl;
1495         } else if (impl instanceof RoutingProvider) {
1496             routingProvider = (RoutingProvider)impl;
1497         } else if (impl instanceof L3ForwardingProvider) {
1498             l3ForwardingProvider = (L3ForwardingProvider)impl;
1499         }else if (impl instanceof GatewayMacResolver) {
1500             gatewayMacResolver = (GatewayMacResolver)impl;
1501         }
1502         populateL3ForwardingCaches();
1503     }
1504 }