openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import java.net.InetAddress;
  12 import java.net.UnknownHostException;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.ExecutorService;
  20 import java.util.concurrent.Executors;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  25 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  26 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  27 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  41 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  42 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  43 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  44 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  49 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  50 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  51 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  52 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  53 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  54 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  55 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  56 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  57 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  58 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  59 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  69 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  70 import org.osgi.framework.ServiceReference;
  71 import org.slf4j.Logger;
  72 import org.slf4j.LoggerFactory;
  73
  74 import com.google.common.base.Preconditions;
  75 import com.google.common.util.concurrent.FutureCallback;
  76 import com.google.common.util.concurrent.Futures;
  77 import com.google.common.util.concurrent.ListenableFuture;
  78
  79 /**
  80  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  81  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  82  * as well as the multi-tenant router forwarding provider.
  83  */
  84 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  85     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  86
  87     // The implementation for each of these services is resolved by the OSGi Service Manager
  88     private volatile ConfigurationService configurationService;
  89     private volatile TenantNetworkManager tenantNetworkManager;
  90     private volatile NodeCacheManager nodeCacheManager;
  91     private volatile INeutronNetworkCRUD neutronNetworkCache;
  92     private volatile INeutronSubnetCRUD neutronSubnetCache;
  93     private volatile INeutronPortCRUD neutronPortCache;
  94     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  95     private volatile L3ForwardingProvider l3ForwardingProvider;
  96     private volatile InboundNatProvider inboundNatProvider;
  97     private volatile OutboundNatProvider outboundNatProvider;
  98     private volatile ArpProvider arpProvider;
  99     private volatile RoutingProvider routingProvider;
 100     private volatile GatewayMacResolver gatewayMacResolver;
 101     private volatile SecurityServicesManager securityServicesManager;
 102
 103     private class FloatIpData {
 104         // br-int of node where floating ip is associated with tenant port
 105         private final Long dpid;
 106         // patch port in br-int used to reach br-ex
 107         private final Long ofPort;
 108         // segmentation id of the net where fixed ip is instantiated
 109         private final String segId;
 110         // mac address assigned to neutron port of floating ip
 111         private final String macAddress;
 112         private final String floatingIpAddress;
 113         // ip address given to tenant vm
 114         private final String fixedIpAddress;
 115         private final String neutronRouterMac;
 116
 117         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 118                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 119             this.dpid = dpid;
 120             this.ofPort = ofPort;
 121             this.segId = segId;
 122             this.macAddress = macAddress;
 123             this.floatingIpAddress = floatingIpAddress;
 124             this.fixedIpAddress = fixedIpAddress;
 125             this.neutronRouterMac = neutronRouterMac;
 126         }
 127     }
 128
 129     private Map<String, String> networkIdToRouterMacCache;
 130     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 131     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 132
 133     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 134     private Map<String, FloatIpData> floatIpDataMapCache;
 135
 136     private String externalRouterMac;
 137     private Boolean enabled = false;
 138     private Boolean flgDistributedARPEnabled = true;
 139     private Boolean isCachePopulationDone = false;
 140     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 141
 142     private Southbound southbound;
 143     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 144
 145     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 146     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 147     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 148     private static final String OWNER_FLOATING_IP = "network:floatingip";
 149     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 150
 151     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 152         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 153         this.neutronModelsDataStoreHelper = neutronHelper;
 154     }
 155
 156     private void initL3AdapterMembers() {
 157         Preconditions.checkNotNull(configurationService);
 158
 159         if (configurationService.isL3ForwardingEnabled()) {
 160             this.networkIdToRouterMacCache = new HashMap<>();
 161             this.networkIdToRouterIpListCache = new HashMap<>();
 162             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 163             this.neutronPortToDpIdCache = new HashMap<>();
 164             this.floatIpDataMapCache = new HashMap<>();
 165
 166             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 167             if (this.externalRouterMac == null) {
 168                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 169             }
 170
 171             this.enabled = true;
 172             LOG.info("OVSDB L3 forwarding is enabled");
 173             if (configurationService.isDistributedArpDisabled()) {
 174                 this.flgDistributedARPEnabled = false;
 175                 LOG.info("Distributed ARP responder is disabled");
 176             } else {
 177                 LOG.debug("Distributed ARP responder is enabled");
 178             }
 179         } else {
 180             LOG.debug("OVSDB L3 forwarding is disabled");
 181         }
 182     }
 183
 184     //
 185     // Callbacks from AbstractHandler
 186     //
 187     @Override
 188     public void processEvent(AbstractEvent abstractEvent) {
 189         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 190             LOG.error("Unable to process abstract event " + abstractEvent);
 191             return;
 192         }
 193         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 194         switch (ev.getAction()) {
 195             case UPDATE:
 196                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 197                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 198                 } else {
 199                     LOG.warn("Received update for an unexpected event " + ev);
 200                 }
 201                 break;
 202             case ADD:
 203                 // fall through...
 204                 // break;
 205             case DELETE:
 206                 // fall through...
 207                 // break;
 208             default:
 209                 LOG.warn("Unable to process event " + ev);
 210                 break;
 211         }
 212     }
 213
 214     //
 215     // Callbacks from GatewayMacResolverListener
 216     //
 217
 218     @Override
 219     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 220         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 221                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 222         if (!this.enabled) {
 223             return;
 224         }
 225
 226         if (macAddress == null || macAddress.getValue() == null) {
 227             // TODO: handle cases when mac is null
 228             return;
 229         }
 230
 231         //
 232         // Enqueue event so update is handled by adapter's thread
 233         //
 234         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 235     }
 236
 237     private void populateL3ForwardingCaches() {
 238         if (!this.enabled) {
 239             return;
 240         }
 241         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 242                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 243             return;
 244         }
 245         this.isCachePopulationDone = true;
 246         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 247         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 248         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 249         if(routers != null && routers.getRouter() != null && ports != null) {
 250             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 251             for( Router router : routers.getRouter()) {
 252                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 253                 if(!ports.getPort().isEmpty()) {
 254                     for( Port port : ports.getPort()) {
 255                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 256                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 257                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 258                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 259                                     , port.getMacAddress());
 260
 261                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 262                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 263                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 264                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 265                         }
 266                     }
 267                 }else {
 268                     LOG.warn("L3 Cache Population :Did not find any port information " +
 269                             "in config Data Store for router {}",router);
 270                 }
 271             }
 272         }
 273         LOG.debug("NetVirt L3 caches population is done");
 274     }
 275
 276     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 277         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 278             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 279             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 280             for(Node bridge : bridges) {
 281                 List<OvsdbTerminationPointAugmentation> interfaces
 282                         = southbound.extractTerminationPointAugmentations(bridge);
 283                 if(interfaces != null && !interfaces.isEmpty()) {
 284                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 285                             interfaces.size(), bridge.getNodeId());
 286                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 287                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 288                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 289                             Long dpId = getDpidForIntegrationBridge(bridge);
 290                             Uuid interfaceUuid = intf.getInterfaceUuid();
 291                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 292                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 293                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 294                             break;
 295                         }
 296                     }
 297                 }
 298             }
 299         }
 300         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 301     }
 302
 303     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 304         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 305         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 306         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 307             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 308                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 309                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 310                             "Fetching from data store.",neutronFloatingIP.getID());
 311                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 312                 }
 313             }
 314         }
 315         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 316         return floatIpDataMapCache.values();
 317     }
 318     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 319         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 320                 "floating id {} ",neutronFloatingId);
 321         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 322             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 323             if (neutronFloatingIP == null) {
 324                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 325                 return null;
 326             }
 327             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 328             NeutronPort neutronPortForFloatIp = null;
 329             for (NeutronPort neutronPort : neutronPorts) {
 330                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 331                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 332                     neutronPortForFloatIp = neutronPort;
 333                     break;
 334                 }
 335             }
 336
 337             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 338             if(neutronTenantPortUuid == null) {
 339                 return null;
 340             }
 341             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 342             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 343             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 344             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 345
 346             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 347             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 348                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 349             String providerSegmentationId = tenantNeutronNetwork != null ?
 350                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 351             String neutronRouterMac = tenantNeutronNetwork != null ?
 352                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 353
 354             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 355                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 356                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 357                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 358                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 359                                 "seg {} mac {} rtrMac {}",
 360                         fixedIpAddress,
 361                         floatingIpAddress,
 362                         neutronPortForFloatIp,
 363                         neutronTenantPortUuid,
 364                         providerSegmentationId,
 365                         floatingIpMac,
 366                         neutronRouterMac);
 367
 368                 return null;
 369             }
 370
 371             // get ofport for patch port in br-int
 372             final Long dpId = nodeIfPair.getLeft();
 373             final Long ofPort = findOFPortForExtPatch(dpId);
 374             if (ofPort == null) {
 375                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 376                                 "to connect floating ip to external bridge. dpid {}",
 377                         dpId);
 378                 return null;
 379             }
 380
 381             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 382                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 383             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 384
 385         }
 386         return floatIpDataMapCache.get(neutronFloatingId);
 387     }
 388     /**
 389      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 390      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 391      * subscribe so that the mac address associated to the gateway ip address is available.
 392      *
 393      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 394      */
 395     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 396         Preconditions.checkNotNull(externalRouterMacUpdate);
 397
 398         flushExistingIpRewrite();
 399         this.externalRouterMac = externalRouterMacUpdate;
 400         rebuildExistingIpRewrite();
 401     }
 402
 403     /**
 404      * Process the event.
 405      *
 406      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 407      * @param subnet An instance of NeutronSubnet object.
 408      */
 409     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 410         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 411     }
 412
 413     /**
 414      * Process the port event as a router interface event.
 415      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 416      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 417      *
 418      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 419      * @param neutronPort An instance of NeutronPort object.
 420      */
 421     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 422         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 423
 424         this.processSecurityGroupUpdate(neutronPort);
 425         if (!this.enabled) {
 426             return;
 427         }
 428
 429         final boolean isDelete = action == Action.DELETE;
 430
 431         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 432             if(!isDelete){
 433                 Node externalBridgeNode = getExternalBridgeNode();
 434                 if(externalBridgeNode != null){
 435                     LOG.info("Port {} is network router gateway interface, "
 436                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 437                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 438                 }else{
 439                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 440                 }
 441             }else{
 442                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 443
 444                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 445                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 446                     // TODO support IPv6
 447                     if (externalSubnet != null &&
 448                             externalSubnet.getIpVersion() == 4) {
 449                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 450                     }
 451                 }
 452             }
 453         }
 454
 455         // Treat the port event as a router interface event if the port belongs to router. This is a
 456         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 457         //
 458         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 459             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 460
 461             if (neutronPort.getFixedIPs() != null) {
 462                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 463                     NeutronRouter_Interface neutronRouterInterface =
 464                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 465                     // id of router interface to be same as subnet
 466                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 467                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 468
 469                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 470                 }
 471             }
 472         } else {
 473             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 474             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 475             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 476             // there.
 477             //
 478             if (!isDelete && neutronPort.getFixedIPs() != null) {
 479                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 480                     NeutronRouter_Interface neutronRouterInterface =
 481                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 482                     if (neutronRouterInterface != null) {
 483                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 484                     }
 485                 }
 486             }
 487             this.updateL3ForNeutronPort(neutronPort, isDelete);
 488         }
 489     }
 490
 491     /**
 492      * Process the event.
 493      *
 494      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 495      * @param neutronRouter An instance of NeutronRouter object.
 496      */
 497     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 498         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 499     }
 500
 501     /**
 502      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 503      * delete the ports on the same subnet.
 504      *
 505      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 506      * @param neutronRouter An instance of NeutronRouter object.
 507      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 508      */
 509     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 510                                                   final NeutronRouter_Interface neutronRouterInterface,
 511                                                   Action action) {
 512         LOG.debug("Router interface {} got event {}. Subnet {}",
 513                      neutronRouterInterface.getPortUUID(),
 514                      action,
 515                      neutronRouterInterface.getSubnetUUID());
 516         if (!this.enabled) {
 517             return;
 518         }
 519
 520         final boolean isDelete = action == Action.DELETE;
 521
 522         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 523
 524         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 525         // see if they are affected by l3
 526         //
 527         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 528             boolean currPortShouldBeDeleted = false;
 529             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 530             if (isDelete) {
 531                 if (neutronPort.getFixedIPs() != null) {
 532                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 533                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 534                             currPortShouldBeDeleted = true;
 535                             break;
 536                         }
 537                     }
 538                 }
 539             }
 540             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 541         }
 542
 543         if (isDelete) {
 544             /*
 545              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 546              */
 547             this.cleanupRouterCache(neutronRouterInterface);
 548         }
 549     }
 550
 551     /**
 552      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 553      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 554      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 555      * way around) on OpenFlow Table 100.
 556      *
 557      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 558      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 559      */
 560     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 561                                              Action actionIn) {
 562         Preconditions.checkNotNull(neutronFloatingIP);
 563
 564         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 565                 neutronFloatingIP.getFixedIPAddress(),
 566                 neutronFloatingIP.getFloatingIPAddress(),
 567                 neutronFloatingIP.getFloatingNetworkUUID());
 568         if (!this.enabled) {
 569             return;
 570         }
 571
 572         Action action;
 573
 574         // Consider action to be delete if getFixedIPAddress is null
 575         //
 576         if (neutronFloatingIP.getFixedIPAddress() == null) {
 577             action = Action.DELETE;
 578         } else {
 579             action = actionIn;
 580         }
 581
 582         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 583
 584         if (action != Action.DELETE) {
 585             // must be first, as it updates floatIpDataMapCache
 586             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 587
 588             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 589             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 590         } else {
 591             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 592             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 593
 594             // must be last, as it updates floatIpDataMapCache
 595             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 596         }
 597     }
 598
 599     /**
 600      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 601      * ip, otherwise for newer one.
 602      */
 603     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 604         Preconditions.checkNotNull(neutronFloatingIP);
 605
 606         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 607         if (fid == null) {
 608             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 609                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 610             return;
 611         }
 612         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 613                                       action);
 614     }
 615
 616     /**
 617      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 618      * ip, otherwise for newer one.
 619      */
 620     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 621         Preconditions.checkNotNull(neutronFloatingIP);
 622
 623         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 624         if (fid == null) {
 625             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 626                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 627             return;
 628         }
 629         programOutboundIpRewriteStage1(fid, action);
 630     }
 631
 632     private void flushExistingIpRewrite() {
 633         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 634             programOutboundIpRewriteStage1(fid, Action.DELETE);
 635         }
 636     }
 637
 638     private void rebuildExistingIpRewrite() {
 639         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 640             programOutboundIpRewriteStage1(fid, Action.ADD);
 641         }
 642     }
 643
 644     /**
 645      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 646      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 647      * requests.
 648      */
 649     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 650         Preconditions.checkNotNull(neutronFloatingIP);
 651         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 652         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 653
 654         // find bridge Node where floating ip is configured by looking up cache for its port
 655         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 656         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 657         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 658         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 659         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 660         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 661
 662         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 663         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 664                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 665         final String providerSegmentationId = tenantNeutronNetwork != null ?
 666                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 667         final String neutronRouterMac = tenantNeutronNetwork != null ?
 668                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 669
 670         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 671                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 672                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 673                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 674             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 675                     fixedIpAddress,
 676                     floatingIpAddress,
 677                     neutronPortForFloatIp,
 678                     neutronTenantPortUuid,
 679                     providerSegmentationId,
 680                     floatingIpMac,
 681                     neutronRouterMac);
 682             return;
 683         }
 684
 685         // get ofport for patch port in br-int
 686         final Long dpId = nodeIfPair.getLeft();
 687         final Long ofPort = findOFPortForExtPatch(dpId);
 688         if (ofPort == null) {
 689             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 690                     dpId);
 691             return;
 692         }
 693
 694         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 695         //
 696         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 697                 Action.ADD)) {
 698             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 699                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 700             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 701             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 702                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 703                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 704         }
 705     }
 706
 707     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 708         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 709         if (floatIpData == null) {
 710             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 711             return;
 712         }
 713
 714         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 715                 floatIpData.floatingIpAddress, Action.DELETE)) {
 716             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 717             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 718                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 719         }
 720     }
 721
 722     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 723         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 724             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 725                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 726                 return neutronPort;
 727             }
 728         }
 729         return null;
 730     }
 731
 732     private Long findOFPortForExtPatch(Long dpId) {
 733         final String brInt = configurationService.getIntegrationBridgeName();
 734         final String brExt = configurationService.getExternalBridgeName();
 735         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 736
 737         Preconditions.checkNotNull(dpId);
 738         Preconditions.checkNotNull(portNameInt);
 739
 740         final long dpidPrimitive = dpId;
 741         for (Node node : nodeCacheManager.getBridgeNodes()) {
 742             if (dpidPrimitive == southbound.getDataPathId(node)) {
 743                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 744                         southbound.getTerminationPointOfBridge(node, portNameInt);
 745                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 746             }
 747         }
 748         return null;
 749     }
 750
 751     /**
 752      * Process the event.
 753      *
 754      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 755      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 756      */
 757     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 758         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 759     }
 760
 761     //
 762     // Callbacks from OVSDB's southbound handler
 763     //
 764     /**
 765      * Process the event.
 766      *
 767      * @param bridgeNode An instance of Node object.
 768      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 769      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 770      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 771      * object.
 772      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 773      */
 774     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 775                                      final NeutronNetwork neutronNetwork, Action action) {
 776         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 777                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 778         if (!this.enabled) {
 779             return;
 780         }
 781
 782         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 783         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 784         final Uuid interfaceUuid = intf.getInterfaceUuid();
 785
 786         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 787                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 788
 789         if (neutronPort != null) {
 790             final String neutronPortUuid = neutronPort.getPortUUID();
 791
 792             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 793                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 794             }
 795
 796             handleNeutronPortEvent(neutronPort, action);
 797         }
 798
 799         if (action == Action.DELETE && interfaceUuid != null) {
 800             handleInterfaceEventDelete(intf, dpId);
 801         }
 802     }
 803
 804     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 805         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 806         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 807                 neutronPortUuid, dpId, interfaceUuid.getValue());
 808     }
 809
 810     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 811         // Remove entry from neutronPortToDpIdCache based on interface uuid
 812         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 813             final String currPortUuid = entry.getKey();
 814             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 815                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 816                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 817                 neutronPortToDpIdCache.remove(currPortUuid);
 818                 break;
 819             }
 820         }
 821     }
 822
 823     //
 824     // Internal helpers
 825     //
 826     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 827
 828         final String networkUUID = neutronPort.getNetworkUUID();
 829         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 830
 831         // If there is no router interface handling the networkUUID, we are done
 832         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 833             return;
 834         }
 835
 836         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 837         // router interface are handled via handleNeutronRouterInterfaceEvent.
 838         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 839             return;
 840         }
 841
 842         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 843         final String providerSegmentationId = neutronNetwork != null ?
 844                                               neutronNetwork.getProviderSegmentationID() : null;
 845         final String tenantMac = neutronPort.getMacAddress();
 846
 847         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 848             tenantMac == null || tenantMac.isEmpty()) {
 849             // done: go no further w/out all the info needed...
 850             return;
 851         }
 852
 853         final Action action = isDelete ? Action.DELETE : Action.ADD;
 854         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 855         if (nodes.isEmpty()) {
 856             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 857         }
 858         for (Node node : nodes) {
 859             final Long dpid = getDpidForIntegrationBridge(node);
 860             if (dpid == null) {
 861                 continue;
 862             }
 863             if (neutronPort.getFixedIPs() == null) {
 864                 continue;
 865             }
 866             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 867                 final String tenantIpStr = neutronIP.getIpAddress();
 868                 if (tenantIpStr.isEmpty()) {
 869                     continue;
 870                 }
 871
 872                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 873                 // still needed when routing to subnets non-local to node (bug 2076).
 874                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 875
 876                 // Configure distributed ARP responder
 877                 if (flgDistributedARPEnabled) {
 878                     // Arp rule is only needed when segmentation exists in the given node (bug 4752).
 879                     boolean arpNeeded = tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 880                     final Action actionForNode = arpNeeded ? action : Action.DELETE;
 881                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 882                 }
 883             }
 884         }
 885     }
 886
 887     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 888         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 889         /**
 890          * Get updated data and original data for the the changed. Identify the security groups that got
 891          * added and removed and call the appropriate providers for updating the flows.
 892          */
 893         try {
 894             NeutronPort originalPort = neutronPort.getOriginalPort();
 895             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 896                                                                             neutronPort.getOriginalPort());
 897             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 898                                                                               neutronPort);
 899
 900             if (null != addedGroup && !addedGroup.isEmpty()) {
 901                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 902             }
 903             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 904                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 905             }
 906
 907         } catch (Exception e) {
 908             LOG.error("Exception in processSecurityGroupUpdate", e);
 909         }
 910     }
 911
 912     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 913         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 914         if (port1 == null) {
 915             return null;
 916         }
 917         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 918         if (port2 == null) {
 919             return list1;
 920         }
 921         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 922         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 923             NeutronSecurityGroup securityGroup1 = iterator.next();
 924             for (NeutronSecurityGroup securityGroup2 :list2) {
 925                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 926                     iterator.remove();
 927                 }
 928             }
 929         }
 930         return list1;
 931     }
 932
 933     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 934                                            String macAddress, String ipStr,
 935                                            Action actionForNode) {
 936         if (actionForNode == Action.DELETE) {
 937             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 938                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 939         }
 940         if (actionForNode == Action.ADD) {
 941             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 942                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 943         }
 944
 945         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 946                                                        macAddress, ipStr, actionForNode);
 947     }
 948
 949     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 950                                              String macAddress,
 951                                              String address,
 952                                              Action actionForNode) {
 953         Status status;
 954         try {
 955             InetAddress inetAddress = InetAddress.getByName(address);
 956             status = l3ForwardingProvider == null ?
 957                      new Status(StatusCode.SUCCESS) :
 958                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 959                                                                       inetAddress, macAddress, actionForNode);
 960         } catch (UnknownHostException e) {
 961             status = new Status(StatusCode.BADREQUEST);
 962         }
 963
 964         if (status.isSuccess()) {
 965             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 966                          l3ForwardingProvider == null ? "skipped" : "programmed",
 967                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 968         } else {
 969             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 970                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 971         }
 972         return status;
 973     }
 974
 975     // --
 976
 977     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 978                                                        Boolean isDelete) {
 979         Preconditions.checkNotNull(destNeutronRouterInterface);
 980
 981         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 982         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 983         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 984         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 985         final NeutronNetwork neutronNetwork = subnet != null ?
 986                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 987         final String destinationSegmentationId = neutronNetwork != null ?
 988                                                  neutronNetwork.getProviderSegmentationID() : null;
 989         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 990         final String cidr = subnet != null ? subnet.getCidr() : null;
 991         final int mask = getMaskLenFromCidr(cidr);
 992
 993         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 994                      destNeutronRouterInterface, isDelete);
 995
 996         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 997         // the local cache
 998         if (neutronNetwork != null) {
 999             if (macAddress == null || macAddress.isEmpty()) {
1000                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1001             }
1002             if (ipList == null || ipList.isEmpty()) {
1003                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1004             }
1005         }
1006
1007         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1008             cidr == null || cidr.isEmpty() ||
1009             macAddress == null || macAddress.isEmpty() ||
1010             ipList == null || ipList.isEmpty()) {
1011             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1012                          destinationSegmentationId, cidr, macAddress, ipList);
1013             // done: go no further w/out all the info needed...
1014             return;
1015         }
1016
1017         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1018
1019         // Keep cache for finding router's mac from network uuid -- add
1020         //
1021         if (! isDelete) {
1022             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1023             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1024             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1025         }
1026
1027         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1028         if (nodes.isEmpty()) {
1029             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1030         }
1031         for (Node node : nodes) {
1032             final Long dpid = getDpidForIntegrationBridge(node);
1033             if (dpid == null) {
1034                 continue;
1035             }
1036
1037             for (Neutron_IPs neutronIP : ipList) {
1038                 final String ipStr = neutronIP.getIpAddress();
1039                 if (ipStr.isEmpty()) {
1040                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1041                             node.getNodeId().getValue(), ipStr);
1042                     continue;
1043                 }
1044
1045                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1046                 //
1047                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1048                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1049                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1050                                                               neutronNetwork, destinationSegmentationId,
1051                                                               macAddress, ipStr, mask, actionForNode,
1052                                                               true /*isReflexsive*/);
1053                 }
1054
1055                 if (! isExternal) {
1056                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1057                             actionForNode);
1058                 }
1059                 // Enable ARP responder by default, because router interface needs to be responded always.
1060                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1061             }
1062
1063             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1064             // for the external neutron networks.
1065             //
1066             {
1067                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1068                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1069             }
1070         }
1071
1072         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1073     }
1074
1075     private void programFlowForNetworkFromExternal(final Node node,
1076                                                    final Long dpid,
1077                                                    final String destinationSegmentationId,
1078                                                    final String dstMacAddress,
1079                                                    final String destIpStr,
1080                                                    final int destMask,
1081                                                    final Action actionForNode) {
1082         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1083                 dstMacAddress, destIpStr, destMask, actionForNode);
1084     }
1085
1086     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1087                                                            final Long dpid,
1088                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1089                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1090                                                            final NeutronNetwork dstNeutronNetwork,
1091                                                            final String destinationSegmentationId,
1092                                                            final String dstMacAddress,
1093                                                            final String destIpStr,
1094                                                            final int destMask,
1095                                                            final Action actionForNode,
1096                                                            Boolean isReflexsive) {
1097         Preconditions.checkNotNull(srcNeutronRouterInterface);
1098         Preconditions.checkNotNull(dstNeutronRouterInterface);
1099
1100         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1101         if (sourceSubnetId == null) {
1102             LOG.error("Could not get provider Subnet ID from router interface {}",
1103                          srcNeutronRouterInterface.getID());
1104             return;
1105         }
1106
1107         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1108         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1109         if (sourceNetworkId == null) {
1110             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1111             return;
1112         }
1113
1114         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1115         if (sourceNetwork == null) {
1116             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1117             return;
1118         }
1119
1120         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1121             // Isolate subnets from different tenants within the same router
1122             return;
1123         }
1124         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1125         if (sourceSegmentationId == null) {
1126             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1127             return;
1128         }
1129         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1130             // Skip 'self'
1131             return;
1132         }
1133
1134         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1135                                      dstMacAddress, destIpStr, destMask, actionForNode);
1136
1137         // Flip roles src->dst; dst->src
1138         if (isReflexsive) {
1139             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1140             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1141             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1142             final String cidr2 = sourceSubnet.getCidr();
1143             final int mask2 = getMaskLenFromCidr(cidr2);
1144
1145             if (cidr2 == null || cidr2.isEmpty() ||
1146                 macAddress2 == null || macAddress2.isEmpty() ||
1147                 ipList2 == null || ipList2.isEmpty()) {
1148                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1149                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1150                 // done: go no further w/out all the info needed...
1151                 return;
1152             }
1153
1154             for (Neutron_IPs neutronIP2 : ipList2) {
1155                 final String ipStr2 = neutronIP2.getIpAddress();
1156                 if (ipStr2.isEmpty()) {
1157                     continue;
1158                 }
1159                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1160                                                           srcNeutronRouterInterface,
1161                                                           sourceNetwork, sourceSegmentationId,
1162                                                           macAddress2, ipStr2, mask2, actionForNode,
1163                                                           false /*isReflexsive*/);
1164             }
1165         }
1166     }
1167
1168     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1169                                               String destinationSegmentationId,
1170                                               String macAddress, String ipStr, int mask,
1171                                               Action actionForNode) {
1172         if (actionForNode == Action.DELETE) {
1173             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1174                          " action {}",
1175                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1176                          macAddress, ipStr, mask, actionForNode);
1177         }
1178         if (actionForNode == Action.ADD) {
1179             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1180                          " action {}",
1181                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1182                          macAddress, ipStr, mask, actionForNode);
1183         }
1184
1185         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1186                                                           macAddress, ipStr, mask, actionForNode);
1187     }
1188
1189     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1190                                                 String destinationSegmentationId,
1191                                                 String macAddress,
1192                                                 String address, int mask,
1193                                                 Action actionForNode) {
1194         Status status;
1195         try {
1196             InetAddress inetAddress = InetAddress.getByName(address);
1197             status = routingProvider == null ?
1198                      new Status(StatusCode.SUCCESS) :
1199                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1200                                                             macAddress, inetAddress, mask, actionForNode);
1201         } catch (UnknownHostException e) {
1202             status = new Status(StatusCode.BADREQUEST);
1203         }
1204
1205         if (status.isSuccess()) {
1206             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1207                          routingProvider == null ? "skipped" : "programmed",
1208                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1209                          actionForNode);
1210         } else {
1211             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1212                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1213                          actionForNode, status);
1214         }
1215         return status;
1216     }
1217
1218     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1219                                            String macAddress, String ipStr,
1220                                            Action action) {
1221         if (action == Action.DELETE ) {
1222             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1223                     dpid, segOrOfPort, macAddress, ipStr, action);
1224         }
1225         if (action == Action.ADD) {
1226             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1227                     dpid, segOrOfPort, macAddress, ipStr, action);
1228         }
1229
1230         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1231         return status.isSuccess();
1232     }
1233
1234     private Status programStaticArpStage2(Long dpid,
1235                                           String segOrOfPort,
1236                                           String macAddress,
1237                                           String address,
1238                                           Action action) {
1239         Status status;
1240         try {
1241             InetAddress inetAddress = InetAddress.getByName(address);
1242             status = arpProvider == null ?
1243                      new Status(StatusCode.SUCCESS) :
1244                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1245                                                        macAddress, inetAddress, action);
1246         } catch (UnknownHostException e) {
1247             status = new Status(StatusCode.BADREQUEST);
1248         }
1249
1250         if (status.isSuccess()) {
1251             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1252                          arpProvider == null ? "skipped" : "programmed",
1253                          macAddress, address, dpid, segOrOfPort, action);
1254         } else {
1255             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1256                          macAddress, address, dpid, segOrOfPort, action, status);
1257         }
1258         return status;
1259     }
1260
1261     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1262                                                   String matchAddress, String rewriteAddress,
1263                                                   Action action) {
1264         if (action == Action.DELETE ) {
1265             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1266                     " action {}",
1267                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1268         }
1269         if (action == Action.ADD ) {
1270             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1271                     " action {}",
1272                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1273         }
1274
1275         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1276                 rewriteAddress, action);
1277         return status.isSuccess();
1278     }
1279
1280     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1281                                                  String matchAddress, String rewriteAddress,
1282                                                  Action action) {
1283         Status status;
1284         try {
1285             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1286             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1287             status = inboundNatProvider == null ?
1288                     new Status(StatusCode.SUCCESS) :
1289                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1290                             inetMatchAddress, inetRewriteAddress,
1291                             action);
1292         } catch (UnknownHostException e) {
1293             status = new Status(StatusCode.BADREQUEST);
1294         }
1295
1296         if (status.isSuccess()) {
1297             final boolean isSkipped = inboundNatProvider == null;
1298             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1299                     isSkipped ? "skipped" : "programmed",
1300                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1301         } else {
1302             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1303                          " status:{}",
1304                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1305                     status);
1306         }
1307         return status;
1308     }
1309
1310     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1311                                                  Action actionForRewriteExclusion) {
1312         if (actionForRewriteExclusion == Action.DELETE ) {
1313             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1314                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1315         }
1316         if (actionForRewriteExclusion == Action.ADD) {
1317             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1318                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1319         }
1320
1321         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1322     }
1323
1324     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1325                                                    Action actionForNode) {
1326         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1327                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1328
1329         if (status.isSuccess()) {
1330             final boolean isSkipped = outboundNatProvider == null;
1331             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1332                          isSkipped ? "skipped" : "programmed",
1333                          cidr, node.getNodeId().getValue(), actionForNode);
1334         } else {
1335             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1336                          cidr, node.getNodeId().getValue(), actionForNode, status);
1337         }
1338         return status;
1339     }
1340
1341     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1342
1343         if (action == Action.DELETE) {
1344             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1345                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1346         }
1347         if (action == Action.ADD) {
1348             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1349                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1350         }
1351
1352         this.programOutboundIpRewriteStage2(fid, action);
1353     }
1354
1355     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1356         Status status;
1357         try {
1358             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1359             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1360             status = outboundNatProvider == null ?
1361                     new Status(StatusCode.SUCCESS) :
1362                     outboundNatProvider.programIpRewriteRule(
1363                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1364                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1365         } catch (UnknownHostException e) {
1366             status = new Status(StatusCode.BADREQUEST);
1367         }
1368
1369         if (status.isSuccess()) {
1370             final boolean isSkipped = outboundNatProvider == null;
1371             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1372                             " action {}",
1373                          isSkipped ? "skipped" : "programmed",
1374                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1375         } else {
1376             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1377                          " action {} status:{}",
1378                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1379         }
1380         return status;
1381     }
1382
1383     private int getMaskLenFromCidr(String cidr) {
1384         if (cidr == null) {
1385             return 0;
1386         }
1387         String[] splits = cidr.split("/");
1388         if (splits.length != 2) {
1389             return 0;
1390         }
1391
1392         int result;
1393         try {
1394             result = Integer.parseInt(splits[1].trim());
1395         } catch (NumberFormatException nfe) {
1396             result = 0;
1397         }
1398         return result;
1399     }
1400
1401     private Long getDpidForIntegrationBridge(Node node) {
1402         // Check if node is integration bridge; and only then return its dpid
1403         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1404             return southbound.getDataPathId(node);
1405         }
1406         return null;
1407     }
1408
1409     private Long getDpidForExternalBridge(Node node) {
1410         // Check if node is integration bridge; and only then return its dpid
1411         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1412             return southbound.getDataPathId(node);
1413         }
1414         return null;
1415     }
1416
1417     private Node getExternalBridgeNode(){
1418         //Pickup the first node that has external bridge (br-ex).
1419         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1420         //the external network is reachable from every br-ex
1421         // TODO: Consider other deployment scenario, and thing of better solution.
1422         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1423         for(Node node : allBridges){
1424             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1425                 return node;
1426             }
1427         }
1428         return null;
1429     }
1430
1431     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1432         if (gatewayPort.getFixedIPs() == null) {
1433             return null;
1434         }
1435         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1436             String subnetUUID = neutronIPs.getSubnetUUID();
1437             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1438             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1439                 return extSubnet;
1440             }
1441             if (extSubnet == null) {
1442                 // TODO: when subnet is created, try again.
1443                 LOG.debug("subnet {} in not found", subnetUUID);
1444              }
1445         }
1446         return null;
1447     }
1448
1449      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1450          /*
1451           *  Fix for 4277
1452           *  Remove the router cache only after deleting the neutron
1453           *  port l3 flows.
1454           */
1455          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1456
1457          if (neutronPort != null) {
1458              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1459              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1460              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1461          }
1462      }
1463
1464     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1465
1466         Preconditions.checkNotNull(node);
1467         Preconditions.checkNotNull(gatewayPort);
1468         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1469
1470         if(externalNetwork != null){
1471             if(externalNetwork.isRouterExternal()){
1472                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1473
1474                 // TODO: address IPv6 case.
1475                 if (externalSubnet != null &&
1476                     externalSubnet.getIpVersion() == 4 &&
1477                     gatewayPort.getFixedIPs() != null) {
1478                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1479                     ListenableFuture<MacAddress> gatewayMacAddress =
1480                         gatewayMacResolver.resolveMacAddress(this,
1481                                                              getDpidForExternalBridge(node),
1482                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1483                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1484                                                              new MacAddress(gatewayPort.getMacAddress()),
1485                                                              true);
1486                     if(gatewayMacAddress != null){
1487                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1488                             @Override
1489                             public void onSuccess(MacAddress result) {
1490                                 if(result != null){
1491                                     if(!result.getValue().equals(externalRouterMac)){
1492                                         updateExternalRouterMac(result.getValue());
1493                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1494                                     }
1495                                 }else{
1496                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1497                                 }
1498                             }
1499
1500                             @Override
1501                             public void onFailure(Throwable t) {
1502                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1503                             }
1504                         }, gatewayMacResolverPool);
1505                     }
1506                 } else {
1507                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1508                 }
1509             }
1510         }else{
1511             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1512         }
1513     }
1514
1515     /**
1516      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1517      *
1518      * @param ofPort the OF port number
1519      * @return the string with encoded OF port (example format "OFPort|999")
1520      */
1521     public static String encodeExcplicitOFPort(Long ofPort) {
1522         return "OFPort|" + ofPort.toString();
1523     }
1524
1525     @Override
1526     public void setDependencies(ServiceReference serviceReference) {
1527         eventDispatcher =
1528                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1529         eventDispatcher.eventHandlerAdded(serviceReference, this);
1530         tenantNetworkManager =
1531                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1532         configurationService =
1533                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1534         arpProvider =
1535                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1536         inboundNatProvider =
1537                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1538         outboundNatProvider =
1539                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1540         routingProvider =
1541                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1542         l3ForwardingProvider =
1543                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1544         nodeCacheManager =
1545                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1546         southbound =
1547                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1548         gatewayMacResolver =
1549                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1550         securityServicesManager =
1551                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1552         initL3AdapterMembers();
1553     }
1554
1555     @Override
1556     public void setDependencies(Object impl) {
1557         if (impl instanceof INeutronNetworkCRUD) {
1558             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1559         } else if (impl instanceof INeutronPortCRUD) {
1560             neutronPortCache = (INeutronPortCRUD)impl;
1561         } else if (impl instanceof INeutronSubnetCRUD) {
1562             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1563         } else if (impl instanceof INeutronFloatingIPCRUD) {
1564             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1565         } else if (impl instanceof ArpProvider) {
1566             arpProvider = (ArpProvider)impl;
1567         } else if (impl instanceof InboundNatProvider) {
1568             inboundNatProvider = (InboundNatProvider)impl;
1569         } else if (impl instanceof OutboundNatProvider) {
1570             outboundNatProvider = (OutboundNatProvider)impl;
1571         } else if (impl instanceof RoutingProvider) {
1572             routingProvider = (RoutingProvider)impl;
1573         } else if (impl instanceof L3ForwardingProvider) {
1574             l3ForwardingProvider = (L3ForwardingProvider)impl;
1575         }else if (impl instanceof GatewayMacResolver) {
1576             gatewayMacResolver = (GatewayMacResolver)impl;
1577         }
1578         populateL3ForwardingCaches();
1579     }
1580 }