openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  15 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  16 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  17 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  18 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  19 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.IcmpEchoProvider;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  37 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  38 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  39 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  49 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  50 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  60 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  61 import org.osgi.framework.ServiceReference;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 import java.net.InetAddress;
  66 import java.net.UnknownHostException;
  67 import java.util.ArrayList;
  68 import java.util.Collection;
  69 import java.util.HashMap;
  70 import java.util.HashSet;
  71 import java.util.Iterator;
  72 import java.util.List;
  73 import java.util.Map;
  74 import java.util.Set;
  75
  76 /**
  77  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  78  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  79  * as well as the multi-tenant router forwarding provider.
  80  */
  81 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  82     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  83
  84     // The implementation for each of these services is resolved by the OSGi Service Manager
  85     private volatile ConfigurationService configurationService;
  86     private volatile TenantNetworkManager tenantNetworkManager;
  87     private volatile NodeCacheManager nodeCacheManager;
  88     private volatile INeutronNetworkCRUD neutronNetworkCache;
  89     private volatile INeutronSubnetCRUD neutronSubnetCache;
  90     private volatile INeutronPortCRUD neutronPortCache;
  91     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  92     private volatile L3ForwardingProvider l3ForwardingProvider;
  93     private volatile InboundNatProvider inboundNatProvider;
  94     private volatile OutboundNatProvider outboundNatProvider;
  95     private volatile ArpProvider arpProvider;
  96     private volatile RoutingProvider routingProvider;
  97     private volatile GatewayMacResolver gatewayMacResolver;
  98     private volatile SecurityServicesManager securityServicesManager;
  99     private volatile IcmpEchoProvider icmpEchoProvider;
 100
 101     private class FloatIpData {
 102         // br-int of node where floating ip is associated with tenant port
 103         private final Long dpid;
 104         // patch port in br-int used to reach br-ex
 105         private final Long ofPort;
 106         // segmentation id of the net where fixed ip is instantiated
 107         private final String segId;
 108         // mac address assigned to neutron port of floating ip
 109         private final String macAddress;
 110         private final String floatingIpAddress;
 111         // ip address given to tenant vm
 112         private final String fixedIpAddress;
 113         private final String neutronRouterMac;
 114
 115         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 116                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 117             this.dpid = dpid;
 118             this.ofPort = ofPort;
 119             this.segId = segId;
 120             this.macAddress = macAddress;
 121             this.floatingIpAddress = floatingIpAddress;
 122             this.fixedIpAddress = fixedIpAddress;
 123             this.neutronRouterMac = neutronRouterMac;
 124         }
 125     }
 126
 127     private Map<String, String> networkIdToRouterMacCache;
 128     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 129     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 130
 131     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 132     private Map<String, FloatIpData> floatIpDataMapCache;
 133
 134     private String externalRouterMac;
 135     private Boolean enabled = false;
 136     private Boolean isCachePopulationDone = false;
 137     private Set<NeutronPort> portCleanupCache;
 138
 139     private Southbound southbound;
 140     private DistributedArpService distributedArpService;
 141     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 142
 143     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 144     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 145     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 146     private static final String OWNER_FLOATING_IP = "network:floatingip";
 147     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 148
 149     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 150         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 151         this.neutronModelsDataStoreHelper = neutronHelper;
 152     }
 153
 154     private void initL3AdapterMembers() {
 155         Preconditions.checkNotNull(configurationService);
 156
 157         if (configurationService.isL3ForwardingEnabled()) {
 158             this.networkIdToRouterMacCache = new HashMap<>();
 159             this.networkIdToRouterIpListCache = new HashMap<>();
 160             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 161             this.neutronPortToDpIdCache = new HashMap<>();
 162             this.floatIpDataMapCache = new HashMap<>();
 163
 164             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 165             if (this.externalRouterMac == null) {
 166                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 167             }
 168             this.enabled = true;
 169             LOG.info("OVSDB L3 forwarding is enabled");
 170         } else {
 171             LOG.debug("OVSDB L3 forwarding is disabled");
 172         }
 173         this.portCleanupCache = new HashSet<>();
 174     }
 175
 176     //
 177     // Callbacks from AbstractHandler
 178     //
 179     @Override
 180     public void processEvent(AbstractEvent abstractEvent) {
 181         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 182             LOG.error("Unable to process abstract event " + abstractEvent);
 183             return;
 184         }
 185         if (!this.enabled) {
 186             return;
 187         }
 188
 189         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 190         switch (ev.getAction()) {
 191             case UPDATE:
 192                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 193                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 194                 } else {
 195                     LOG.warn("Received update for an unexpected event " + ev);
 196                 }
 197                 break;
 198             case ADD:
 199                 // fall through...
 200                 // break;
 201             case DELETE:
 202                 // fall through...
 203                 // break;
 204             default:
 205                 LOG.warn("Unable to process event " + ev);
 206                 break;
 207         }
 208     }
 209
 210     //
 211     // Callbacks from GatewayMacResolverListener
 212     //
 213
 214     @Override
 215     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 216         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 217                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 218         if (!this.enabled) {
 219             return;
 220         }
 221
 222         if (macAddress == null || macAddress.getValue() == null) {
 223             // TODO: handle cases when mac is null
 224             return;
 225         }
 226
 227         //
 228         // Enqueue event so update is handled by adapter's thread
 229         //
 230         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 231     }
 232
 233     private void populateL3ForwardingCaches() {
 234         if (!this.enabled) {
 235             return;
 236         }
 237         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 238                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 239             return;
 240         }
 241         this.isCachePopulationDone = true;
 242         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 243         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 244         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 245         if(routers != null && routers.getRouter() != null && ports != null) {
 246             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 247             for( Router router : routers.getRouter()) {
 248                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 249                 if(!ports.getPort().isEmpty()) {
 250                     for( Port port : ports.getPort()) {
 251                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 252                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 253                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 254                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 255                                     , port.getMacAddress());
 256
 257                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 258                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 259                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 260                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 261                         }
 262                     }
 263                 }else {
 264                     LOG.warn("L3 Cache Population :Did not find any port information " +
 265                             "in config Data Store for router {}",router);
 266                 }
 267             }
 268         }
 269         LOG.debug("NetVirt L3 caches population is done");
 270     }
 271
 272     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 273         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 274             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 275             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 276             for(Node bridge : bridges) {
 277                 List<OvsdbTerminationPointAugmentation> interfaces
 278                         = southbound.extractTerminationPointAugmentations(bridge);
 279                 if(interfaces != null && !interfaces.isEmpty()) {
 280                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 281                             interfaces.size(), bridge.getNodeId());
 282                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 283                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 284                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 285                             Long dpId = getDpidForIntegrationBridge(bridge);
 286                             Uuid interfaceUuid = intf.getInterfaceUuid();
 287                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 288                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 289                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 290                             break;
 291                         }
 292                     }
 293                 }
 294             }
 295         }
 296         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 297     }
 298
 299     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 300         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 301         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 302         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 303             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 304                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 305                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 306                             "Fetching from data store.",neutronFloatingIP.getID());
 307                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 308                 }
 309             }
 310         }
 311         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 312         return floatIpDataMapCache.values();
 313     }
 314     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 315         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 316                 "floating id {} ",neutronFloatingId);
 317         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 318             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 319             if (neutronFloatingIP == null) {
 320                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 321                 return null;
 322             }
 323             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 324             NeutronPort neutronPortForFloatIp = null;
 325             for (NeutronPort neutronPort : neutronPorts) {
 326                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 327                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 328                     neutronPortForFloatIp = neutronPort;
 329                     break;
 330                 }
 331             }
 332
 333             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 334             if(neutronTenantPortUuid == null) {
 335                 return null;
 336             }
 337             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 338             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 339             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 340             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 341
 342             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 343             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 344                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 345             String providerSegmentationId = tenantNeutronNetwork != null ?
 346                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 347             String neutronRouterMac = tenantNeutronNetwork != null ?
 348                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 349
 350             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 351                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 352                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 353                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 354                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 355                                 "seg {} mac {} rtrMac {}",
 356                         fixedIpAddress,
 357                         floatingIpAddress,
 358                         neutronPortForFloatIp,
 359                         neutronTenantPortUuid,
 360                         providerSegmentationId,
 361                         floatingIpMac,
 362                         neutronRouterMac);
 363
 364                 return null;
 365             }
 366
 367             // get ofport for patch port in br-int
 368             final Long dpId = nodeIfPair.getLeft();
 369             final Long ofPort = findOFPortForExtPatch(dpId);
 370             if (ofPort == null) {
 371                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 372                                 "to connect floating ip to external bridge. dpid {}",
 373                         dpId);
 374                 return null;
 375             }
 376
 377             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 378                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 379             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 380
 381         }
 382         return floatIpDataMapCache.get(neutronFloatingId);
 383     }
 384     /**
 385      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 386      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 387      * subscribe so that the mac address associated to the gateway ip address is available.
 388      *
 389      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 390      */
 391     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 392         Preconditions.checkNotNull(externalRouterMacUpdate);
 393
 394         flushExistingIpRewrite();
 395         this.externalRouterMac = externalRouterMacUpdate;
 396         rebuildExistingIpRewrite();
 397     }
 398
 399     /**
 400      * Process the event.
 401      *
 402      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 403      * @param subnet An instance of NeutronSubnet object.
 404      */
 405     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 406         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 407     }
 408
 409     /**
 410      * Process the port event as a router interface event.
 411      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 412      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 413      *
 414      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 415      * @param neutronPort An instance of NeutronPort object.
 416      */
 417     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 418         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 419
 420         if (action == Action.UPDATE) {
 421             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 422             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 423             this.processSecurityGroupUpdate(neutronPort);
 424         }
 425
 426         if (!this.enabled) {
 427             return;
 428         }
 429
 430         final boolean isDelete = action == Action.DELETE;
 431
 432         if (action == Action.DELETE) {
 433             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 434             this.cleanupFloatingIPRules(neutronPort);
 435         }
 436
 437         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 438             if (!isDelete) {
 439                 LOG.info("Port {} is network router gateway interface, "
 440                         + "triggering gateway resolution for the attached external network", neutronPort);
 441                 this.triggerGatewayMacResolver(neutronPort);
 442             }else{
 443                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 444
 445                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 446                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 447                     // TODO support IPv6
 448                     if (externalSubnet != null &&
 449                             externalSubnet.getIpVersion() == 4) {
 450                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 451                     }
 452                 }
 453             }
 454         }
 455
 456         // Treat the port event as a router interface event if the port belongs to router. This is a
 457         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 458         //
 459         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 460             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 461
 462             if (neutronPort.getFixedIPs() != null) {
 463                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 464                     NeutronRouter_Interface neutronRouterInterface =
 465                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 466                     // id of router interface to be same as subnet
 467                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 468                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 469
 470                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 471                 }
 472             }
 473         } else {
 474             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 475             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 476             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 477             // there.
 478             //
 479             if (!isDelete && neutronPort.getFixedIPs() != null) {
 480                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 481                     NeutronRouter_Interface neutronRouterInterface =
 482                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 483                     if (neutronRouterInterface != null) {
 484                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 485                     }
 486                 }
 487             }
 488             this.updateL3ForNeutronPort(neutronPort, isDelete);
 489         }
 490     }
 491
 492     /**
 493      * Process the event.
 494      *
 495      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 496      * @param neutronRouter An instance of NeutronRouter object.
 497      */
 498     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 499         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 500     }
 501
 502     /**
 503      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 504      * delete the ports on the same subnet.
 505      *
 506      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 507      * @param neutronRouter An instance of NeutronRouter object.
 508      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 509      */
 510     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 511                                                   final NeutronRouter_Interface neutronRouterInterface,
 512                                                   Action action) {
 513         LOG.debug("Router interface {} got event {}. Subnet {}",
 514                      neutronRouterInterface.getPortUUID(),
 515                      action,
 516                      neutronRouterInterface.getSubnetUUID());
 517         if (!this.enabled) {
 518             return;
 519         }
 520
 521         final boolean isDelete = action == Action.DELETE;
 522
 523         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 524
 525         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 526         // see if they are affected by l3
 527         //
 528         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 529             boolean currPortShouldBeDeleted = false;
 530             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 531             if (isDelete) {
 532                 if (neutronPort.getFixedIPs() != null) {
 533                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 534                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 535                             currPortShouldBeDeleted = true;
 536                             break;
 537                         }
 538                     }
 539                 }
 540             }
 541             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 542         }
 543
 544         if (isDelete) {
 545             /*
 546              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 547              */
 548             this.cleanupRouterCache(neutronRouterInterface);
 549         }
 550     }
 551
 552     /**
 553      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 554      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 555      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 556      * way around) on OpenFlow Table 100.
 557      *
 558      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 559      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 560      */
 561     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 562                                              Action actionIn) {
 563         Preconditions.checkNotNull(neutronFloatingIP);
 564
 565         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 566                 neutronFloatingIP.getFixedIPAddress(),
 567                 neutronFloatingIP.getFloatingIPAddress(),
 568                 neutronFloatingIP.getFloatingNetworkUUID());
 569         if (!this.enabled) {
 570             return;
 571         }
 572
 573         Action action;
 574
 575         // Consider action to be delete if getFixedIPAddress is null
 576         //
 577         if (neutronFloatingIP.getFixedIPAddress() == null) {
 578             action = Action.DELETE;
 579         } else {
 580             action = actionIn;
 581         }
 582
 583         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 584
 585         if (action != Action.DELETE) {
 586             // must be first, as it updates floatIpDataMapCache
 587             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 588
 589             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 590             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 591         } else {
 592             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 593             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 594
 595             // must be last, as it updates floatIpDataMapCache
 596             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 597         }
 598     }
 599
 600     /**
 601      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 602      * ip, otherwise for newer one.
 603      */
 604     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 605         Preconditions.checkNotNull(neutronFloatingIP);
 606
 607         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 608         if (fid == null) {
 609             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 610                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 611             return;
 612         }
 613         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 614                                       action);
 615     }
 616
 617     /**
 618      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 619      * ip, otherwise for newer one.
 620      */
 621     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 622         Preconditions.checkNotNull(neutronFloatingIP);
 623
 624         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 625         if (fid == null) {
 626             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 627                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 628             return;
 629         }
 630         programOutboundIpRewriteStage1(fid, action);
 631     }
 632
 633     private void flushExistingIpRewrite() {
 634         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 635             programOutboundIpRewriteStage1(fid, Action.DELETE);
 636         }
 637     }
 638
 639     private void rebuildExistingIpRewrite() {
 640         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 641             programOutboundIpRewriteStage1(fid, Action.ADD);
 642         }
 643     }
 644
 645     /**
 646      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 647      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 648      * requests.
 649      */
 650     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 651         Preconditions.checkNotNull(neutronFloatingIP);
 652         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 653         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 654
 655         // find bridge Node where floating ip is configured by looking up cache for its port
 656         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 657         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 658         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 659         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 660         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 661         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 662
 663         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 664         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 665                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 666         final String providerSegmentationId = tenantNeutronNetwork != null ?
 667                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 668         final String neutronRouterMac = tenantNeutronNetwork != null ?
 669                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 670
 671         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 672                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 673                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 674                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 675             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 676                     fixedIpAddress,
 677                     floatingIpAddress,
 678                     neutronPortForFloatIp,
 679                     neutronTenantPortUuid,
 680                     providerSegmentationId,
 681                     floatingIpMac,
 682                     neutronRouterMac);
 683             return;
 684         }
 685
 686         // get ofport for patch port in br-int
 687         final Long dpId = nodeIfPair.getLeft();
 688         final Long ofPort = findOFPortForExtPatch(dpId);
 689         if (ofPort == null) {
 690             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 691                     dpId);
 692             return;
 693         }
 694
 695         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 696         //
 697         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 698                 Action.ADD)) {
 699             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 700                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 701             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 702             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 703                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 704                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 705         }
 706     }
 707
 708     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 709         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 710         if (floatIpData == null) {
 711             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 712             return;
 713         }
 714
 715         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 716                 floatIpData.floatingIpAddress, Action.DELETE)) {
 717             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 718             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 719                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 720         }
 721     }
 722
 723     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 724         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 725             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 726                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 727                 return neutronPort;
 728             }
 729         }
 730         return null;
 731     }
 732
 733     private Long findOFPortForExtPatch(Long dpId) {
 734         final String brInt = configurationService.getIntegrationBridgeName();
 735         final String brExt = configurationService.getExternalBridgeName();
 736         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 737
 738         Preconditions.checkNotNull(dpId);
 739         Preconditions.checkNotNull(portNameInt);
 740
 741         final long dpidPrimitive = dpId;
 742         for (Node node : nodeCacheManager.getBridgeNodes()) {
 743             if (dpidPrimitive == southbound.getDataPathId(node)) {
 744                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 745                         southbound.getTerminationPointOfBridge(node, portNameInt);
 746                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 747             }
 748         }
 749         return null;
 750     }
 751
 752     /**
 753      * Process the event.
 754      *
 755      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 756      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 757      */
 758     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 759         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 760     }
 761
 762     //
 763     // Callbacks from OVSDB's southbound handler
 764     //
 765     /**
 766      * Process the event.
 767      *
 768      * @param bridgeNode An instance of Node object.
 769      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 770      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 771      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 772      * object.
 773      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 774      */
 775     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 776                                      final NeutronNetwork neutronNetwork, Action action) {
 777         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 778                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 779
 780         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 781         if (action != Action.DELETE && neutronPort != null) {
 782             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 783             storePortInCleanupCache(neutronPort);
 784         }
 785
 786         if (!this.enabled) {
 787             return;
 788         }
 789
 790         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 791         final Uuid interfaceUuid = intf.getInterfaceUuid();
 792
 793         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 794                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 795
 796         if (neutronPort != null) {
 797             final String neutronPortUuid = neutronPort.getPortUUID();
 798
 799             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 800                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 801             }
 802
 803             handleNeutronPortEvent(neutronPort, action);
 804         }
 805
 806         if (action == Action.DELETE && interfaceUuid != null) {
 807             handleInterfaceEventDelete(intf, dpId);
 808         }
 809     }
 810
 811     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 812         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 813         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 814                 neutronPortUuid, dpId, interfaceUuid.getValue());
 815     }
 816
 817     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 818         // Remove entry from neutronPortToDpIdCache based on interface uuid
 819         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 820             final String currPortUuid = entry.getKey();
 821             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 822                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 823                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 824                 neutronPortToDpIdCache.remove(currPortUuid);
 825                 break;
 826             }
 827         }
 828     }
 829
 830     //
 831     // Internal helpers
 832     //
 833     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 834
 835         final String networkUUID = neutronPort.getNetworkUUID();
 836         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 837
 838         // If there is no router interface handling the networkUUID, we are done
 839         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 840             return;
 841         }
 842
 843         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 844         // router interface are handled via handleNeutronRouterInterfaceEvent.
 845         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 846             return;
 847         }
 848
 849         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 850         final String providerSegmentationId = neutronNetwork != null ?
 851                                               neutronNetwork.getProviderSegmentationID() : null;
 852         final String tenantMac = neutronPort.getMacAddress();
 853
 854         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 855             tenantMac == null || tenantMac.isEmpty()) {
 856             // done: go no further w/out all the info needed...
 857             return;
 858         }
 859
 860         final Action action = isDelete ? Action.DELETE : Action.ADD;
 861         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 862         if (nodes.isEmpty()) {
 863             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 864         }
 865         for (Node node : nodes) {
 866             final Long dpid = getDpidForIntegrationBridge(node);
 867             if (dpid == null) {
 868                 continue;
 869             }
 870             if (neutronPort.getFixedIPs() == null) {
 871                 continue;
 872             }
 873             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 874                 final String tenantIpStr = neutronIP.getIpAddress();
 875                 if (tenantIpStr.isEmpty()) {
 876                     continue;
 877                 }
 878
 879                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 880                 // still needed when routing to subnets non-local to node (bug 2076).
 881                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 882             }
 883         }
 884     }
 885
 886     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 887         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 888         /**
 889          * Get updated data and original data for the the changed. Identify the security groups that got
 890          * added and removed and call the appropriate providers for updating the flows.
 891          */
 892         try {
 893             NeutronPort originalPort = neutronPort.getOriginalPort();
 894             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 895                                                                             neutronPort.getOriginalPort());
 896             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 897                                                                               neutronPort);
 898
 899             if (null != addedGroup && !addedGroup.isEmpty()) {
 900                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 901             }
 902             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 903                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 904             }
 905
 906         } catch (Exception e) {
 907             LOG.error("Exception in processSecurityGroupUpdate", e);
 908         }
 909     }
 910
 911     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 912         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 913         if (port1 == null) {
 914             return null;
 915         }
 916         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 917         if (port2 == null) {
 918             return list1;
 919         }
 920         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 921         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 922             NeutronSecurityGroup securityGroup1 = iterator.next();
 923             for (NeutronSecurityGroup securityGroup2 :list2) {
 924                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 925                     iterator.remove();
 926                 }
 927             }
 928         }
 929         return list1;
 930     }
 931
 932     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 933                                            String macAddress, String ipStr,
 934                                            Action actionForNode) {
 935         if (actionForNode == Action.DELETE) {
 936             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 937                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 938         }
 939         if (actionForNode == Action.ADD) {
 940             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 941                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 942         }
 943
 944         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 945                                                        macAddress, ipStr, actionForNode);
 946     }
 947
 948     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 949                                              String macAddress,
 950                                              String address,
 951                                              Action actionForNode) {
 952         Status status;
 953         try {
 954             InetAddress inetAddress = InetAddress.getByName(address);
 955             status = l3ForwardingProvider == null ?
 956                      new Status(StatusCode.SUCCESS) :
 957                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 958                                                                       inetAddress, macAddress, actionForNode);
 959         } catch (UnknownHostException e) {
 960             status = new Status(StatusCode.BADREQUEST);
 961         }
 962
 963         if (status.isSuccess()) {
 964             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 965                          l3ForwardingProvider == null ? "skipped" : "programmed",
 966                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 967         } else {
 968             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 969                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 970         }
 971         return status;
 972     }
 973
 974     // --
 975
 976     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 977                                                        Boolean isDelete) {
 978         Preconditions.checkNotNull(destNeutronRouterInterface);
 979
 980         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 981         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 982         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 983         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 984         final NeutronNetwork neutronNetwork = subnet != null ?
 985                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 986         final String destinationSegmentationId = neutronNetwork != null ?
 987                                                  neutronNetwork.getProviderSegmentationID() : null;
 988         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 989         final String cidr = subnet != null ? subnet.getCidr() : null;
 990         final int mask = getMaskLenFromCidr(cidr);
 991
 992         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 993                      destNeutronRouterInterface, isDelete);
 994
 995         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 996         // the local cache
 997         if (neutronNetwork != null) {
 998             if (macAddress == null || macAddress.isEmpty()) {
 999                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1000             }
1001             if (ipList == null || ipList.isEmpty()) {
1002                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1003             }
1004         }
1005
1006         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1007             cidr == null || cidr.isEmpty() ||
1008             macAddress == null || macAddress.isEmpty() ||
1009             ipList == null || ipList.isEmpty()) {
1010             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1011                          destinationSegmentationId, cidr, macAddress, ipList);
1012             // done: go no further w/out all the info needed...
1013             return;
1014         }
1015
1016         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1017
1018         // Keep cache for finding router's mac from network uuid -- add
1019         //
1020         if (! isDelete) {
1021             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1022             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1023             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1024         }
1025
1026         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1027         if (nodes.isEmpty()) {
1028             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1029         }
1030         for (Node node : nodes) {
1031             final Long dpid = getDpidForIntegrationBridge(node);
1032             if (dpid == null) {
1033                 continue;
1034             }
1035
1036             for (Neutron_IPs neutronIP : ipList) {
1037                 final String ipStr = neutronIP.getIpAddress();
1038                 if (ipStr.isEmpty()) {
1039                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1040                             node.getNodeId().getValue(), ipStr);
1041                     continue;
1042                 }
1043
1044                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1045                 //
1046                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1047                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1048                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1049                                                               neutronNetwork, destinationSegmentationId,
1050                                                               macAddress, ipStr, mask, actionForNode,
1051                                                               true /*isReflexsive*/);
1052                 }
1053
1054                 if (! isExternal) {
1055                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1056                             actionForNode);
1057                 }
1058                 // Enable ARP responder by default, because router interface needs to be responded always.
1059                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1060                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1061             }
1062
1063             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1064             // for the external neutron networks.
1065             //
1066             {
1067                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1068                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1069             }
1070         }
1071
1072         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1073     }
1074
1075     private void programFlowForNetworkFromExternal(final Node node,
1076                                                    final Long dpid,
1077                                                    final String destinationSegmentationId,
1078                                                    final String dstMacAddress,
1079                                                    final String destIpStr,
1080                                                    final int destMask,
1081                                                    final Action actionForNode) {
1082         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1083                 dstMacAddress, destIpStr, destMask, actionForNode);
1084     }
1085
1086     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1087                                                            final Long dpid,
1088                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1089                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1090                                                            final NeutronNetwork dstNeutronNetwork,
1091                                                            final String destinationSegmentationId,
1092                                                            final String dstMacAddress,
1093                                                            final String destIpStr,
1094                                                            final int destMask,
1095                                                            final Action actionForNode,
1096                                                            Boolean isReflexsive) {
1097         Preconditions.checkNotNull(srcNeutronRouterInterface);
1098         Preconditions.checkNotNull(dstNeutronRouterInterface);
1099
1100         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1101         if (sourceSubnetId == null) {
1102             LOG.error("Could not get provider Subnet ID from router interface {}",
1103                          srcNeutronRouterInterface.getID());
1104             return;
1105         }
1106
1107         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1108         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1109         if (sourceNetworkId == null) {
1110             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1111             return;
1112         }
1113
1114         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1115         if (sourceNetwork == null) {
1116             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1117             return;
1118         }
1119
1120         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1121             // Isolate subnets from different tenants within the same router
1122             return;
1123         }
1124         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1125         if (sourceSegmentationId == null) {
1126             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1127             return;
1128         }
1129         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1130             // Skip 'self'
1131             return;
1132         }
1133
1134         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1135                                      dstMacAddress, destIpStr, destMask, actionForNode);
1136
1137         // Flip roles src->dst; dst->src
1138         if (isReflexsive) {
1139             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1140             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1141             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1142             final String cidr2 = sourceSubnet.getCidr();
1143             final int mask2 = getMaskLenFromCidr(cidr2);
1144
1145             if (cidr2 == null || cidr2.isEmpty() ||
1146                 macAddress2 == null || macAddress2.isEmpty() ||
1147                 ipList2 == null || ipList2.isEmpty()) {
1148                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1149                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1150                 // done: go no further w/out all the info needed...
1151                 return;
1152             }
1153
1154             for (Neutron_IPs neutronIP2 : ipList2) {
1155                 final String ipStr2 = neutronIP2.getIpAddress();
1156                 if (ipStr2.isEmpty()) {
1157                     continue;
1158                 }
1159                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1160                                                           srcNeutronRouterInterface,
1161                                                           sourceNetwork, sourceSegmentationId,
1162                                                           macAddress2, ipStr2, mask2, actionForNode,
1163                                                           false /*isReflexsive*/);
1164             }
1165         }
1166     }
1167
1168     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1169                                               String destinationSegmentationId,
1170                                               String macAddress, String ipStr, int mask,
1171                                               Action actionForNode) {
1172         if (actionForNode == Action.DELETE) {
1173             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1174                          " action {}",
1175                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1176                          macAddress, ipStr, mask, actionForNode);
1177         }
1178         if (actionForNode == Action.ADD) {
1179             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1180                          " action {}",
1181                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1182                          macAddress, ipStr, mask, actionForNode);
1183         }
1184
1185         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1186                                                           macAddress, ipStr, mask, actionForNode);
1187     }
1188
1189     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1190                                                 String destinationSegmentationId,
1191                                                 String macAddress,
1192                                                 String address, int mask,
1193                                                 Action actionForNode) {
1194         Status status;
1195         try {
1196             InetAddress inetAddress = InetAddress.getByName(address);
1197             status = routingProvider == null ?
1198                      new Status(StatusCode.SUCCESS) :
1199                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1200                                                             macAddress, inetAddress, mask, actionForNode);
1201         } catch (UnknownHostException e) {
1202             status = new Status(StatusCode.BADREQUEST);
1203         }
1204
1205         if (status.isSuccess()) {
1206             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1207                          routingProvider == null ? "skipped" : "programmed",
1208                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1209                          actionForNode);
1210         } else {
1211             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1212                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1213                          actionForNode, status);
1214         }
1215         return status;
1216     }
1217
1218     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1219                                            String macAddress, String ipStr,
1220                                            Action action) {
1221         if (action == Action.DELETE ) {
1222             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1223                     dpid, segOrOfPort, macAddress, ipStr, action);
1224         }
1225         if (action == Action.ADD) {
1226             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1227                     dpid, segOrOfPort, macAddress, ipStr, action);
1228         }
1229
1230         Status status = new Status(StatusCode.UNSUPPORTED);
1231         if (icmpEchoProvider != null){
1232             try {
1233                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1234                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1235                                                 macAddress, inetAddress, action);
1236             } catch (UnknownHostException e) {
1237                 status = new Status(StatusCode.BADREQUEST);
1238             }
1239         }
1240
1241         if (status.isSuccess()) {
1242             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1243                     icmpEchoProvider == null ? "skipped" : "programmed",
1244                     macAddress, ipStr, dpid, segOrOfPort, action);
1245         } else {
1246             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1247                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1248         }
1249
1250         return status.isSuccess();
1251     }
1252
1253     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1254                                                   String matchAddress, String rewriteAddress,
1255                                                   Action action) {
1256         if (action == Action.DELETE ) {
1257             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1258                     " action {}",
1259                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1260         }
1261         if (action == Action.ADD ) {
1262             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1263                     " action {}",
1264                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1265         }
1266
1267         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1268                 rewriteAddress, action);
1269         return status.isSuccess();
1270     }
1271
1272     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1273                                                  String matchAddress, String rewriteAddress,
1274                                                  Action action) {
1275         Status status;
1276         try {
1277             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1278             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1279             status = inboundNatProvider == null ?
1280                     new Status(StatusCode.SUCCESS) :
1281                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1282                             inetMatchAddress, inetRewriteAddress,
1283                             action);
1284         } catch (UnknownHostException e) {
1285             status = new Status(StatusCode.BADREQUEST);
1286         }
1287
1288         if (status.isSuccess()) {
1289             final boolean isSkipped = inboundNatProvider == null;
1290             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1291                     isSkipped ? "skipped" : "programmed",
1292                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1293         } else {
1294             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1295                          " status:{}",
1296                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1297                     status);
1298         }
1299         return status;
1300     }
1301
1302     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1303                                                  Action actionForRewriteExclusion) {
1304         if (actionForRewriteExclusion == Action.DELETE ) {
1305             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1306                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1307         }
1308         if (actionForRewriteExclusion == Action.ADD) {
1309             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1310                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1311         }
1312
1313         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1314     }
1315
1316     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1317                                                    Action actionForNode) {
1318         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1319                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1320
1321         if (status.isSuccess()) {
1322             final boolean isSkipped = outboundNatProvider == null;
1323             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1324                          isSkipped ? "skipped" : "programmed",
1325                          cidr, node.getNodeId().getValue(), actionForNode);
1326         } else {
1327             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1328                          cidr, node.getNodeId().getValue(), actionForNode, status);
1329         }
1330         return status;
1331     }
1332
1333     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1334
1335         if (action == Action.DELETE) {
1336             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1337                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1338         }
1339         if (action == Action.ADD) {
1340             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1341                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1342         }
1343
1344         this.programOutboundIpRewriteStage2(fid, action);
1345     }
1346
1347     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1348         Status status;
1349         try {
1350             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1351             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1352             status = outboundNatProvider == null ?
1353                     new Status(StatusCode.SUCCESS) :
1354                     outboundNatProvider.programIpRewriteRule(
1355                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1356                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1357         } catch (UnknownHostException e) {
1358             status = new Status(StatusCode.BADREQUEST);
1359         }
1360
1361         if (status.isSuccess()) {
1362             final boolean isSkipped = outboundNatProvider == null;
1363             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1364                             " action {}",
1365                          isSkipped ? "skipped" : "programmed",
1366                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1367         } else {
1368             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1369                          " action {} status:{}",
1370                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1371         }
1372         return status;
1373     }
1374
1375     private int getMaskLenFromCidr(String cidr) {
1376         if (cidr == null) {
1377             return 0;
1378         }
1379         String[] splits = cidr.split("/");
1380         if (splits.length != 2) {
1381             return 0;
1382         }
1383
1384         int result;
1385         try {
1386             result = Integer.parseInt(splits[1].trim());
1387         } catch (NumberFormatException nfe) {
1388             result = 0;
1389         }
1390         return result;
1391     }
1392
1393     private Long getDpidForIntegrationBridge(Node node) {
1394         // Check if node is integration bridge; and only then return its dpid
1395         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1396             return southbound.getDataPathId(node);
1397         }
1398         return null;
1399     }
1400
1401     private Long getDpidForExternalBridge(Node node) {
1402         // Check if node is external bridge; and only then return its dpid
1403         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1404             return southbound.getDataPathId(node);
1405         }
1406         return null;
1407     }
1408
1409     private Node getExternalBridgeNode(){
1410         //Pickup the first node that has external bridge (br-ex).
1411         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1412         //the external network is reachable from every br-ex
1413         // TODO: Consider other deployment scenario, and thing of better solution.
1414         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1415         for(Node node : allBridges){
1416             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1417                 return node;
1418             }
1419         }
1420         return null;
1421     }
1422
1423     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1424         if (gatewayPort.getFixedIPs() == null) {
1425             return null;
1426         }
1427         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1428             String subnetUUID = neutronIPs.getSubnetUUID();
1429             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1430             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1431                 return extSubnet;
1432             }
1433             if (extSubnet == null) {
1434                 // TODO: when subnet is created, try again.
1435                 LOG.debug("subnet {} in not found", subnetUUID);
1436              }
1437         }
1438         return null;
1439     }
1440
1441      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1442          /*
1443           *  Fix for 4277
1444           *  Remove the router cache only after deleting the neutron
1445           *  port l3 flows.
1446           */
1447          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1448
1449          if (neutronPort != null) {
1450              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1451              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1452              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1453          }
1454      }
1455
1456     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1457
1458         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1459         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1460             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1461                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1462                     handleNeutronFloatingIPEvent(neutronFloatingIP, Action.DELETE);
1463                 }
1464             }
1465         }
1466     }
1467
1468
1469     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1470
1471         Preconditions.checkNotNull(gatewayPort);
1472         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1473
1474         if(externalNetwork != null){
1475             if(externalNetwork.isRouterExternal()){
1476                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1477
1478                 // TODO: address IPv6 case.
1479                 if (externalSubnet != null &&
1480                         externalSubnet.getIpVersion() == 4 &&
1481                         gatewayPort.getFixedIPs() != null) {
1482                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1483
1484                     gatewayMacResolver.resolveMacAddress(
1485                             this, /* gatewayMacResolverListener */
1486                             null, /* externalNetworkBridgeDpid */
1487                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1488                             new Ipv4Address(externalSubnet.getGatewayIP()),
1489                             new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1490                             new MacAddress(gatewayPort.getMacAddress()),
1491                             true /* periodicRefresh */);
1492                 } else {
1493                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1494                 }
1495             }
1496         }else{
1497             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1498         }
1499     }
1500
1501
1502     private void storePortInCleanupCache(NeutronPort port) {
1503         this.portCleanupCache.add(port);
1504     }
1505
1506
1507     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1508         removePortFromCleanupCache(originalPort);
1509         storePortInCleanupCache(updatedPort);
1510     }
1511
1512     public void removePortFromCleanupCache(NeutronPort port) {
1513         this.portCleanupCache.remove(port);
1514     }
1515
1516     public Set<NeutronPort> getPortCleanupCache() {
1517         return this.portCleanupCache;
1518     }
1519
1520     public NeutronPort getPortFromCleanupCache(String portid) {
1521         for (NeutronPort neutronPort : this.portCleanupCache) {
1522             if (neutronPort.getPortUUID() != null ) {
1523                 if (neutronPort.getPortUUID().equals(portid)) {
1524                     LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1525                     return neutronPort;
1526                 }
1527             }
1528         }
1529         return null;
1530     }
1531
1532     /**
1533      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1534      *
1535      * @param ofPort the OF port number
1536      * @return the string with encoded OF port (example format "OFPort|999")
1537      */
1538     public static String encodeExcplicitOFPort(Long ofPort) {
1539         return "OFPort|" + ofPort.toString();
1540     }
1541
1542     @Override
1543     public void setDependencies(ServiceReference serviceReference) {
1544         eventDispatcher =
1545                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1546         eventDispatcher.eventHandlerAdded(serviceReference, this);
1547         tenantNetworkManager =
1548                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1549         configurationService =
1550                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1551         arpProvider =
1552                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1553         inboundNatProvider =
1554                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1555         outboundNatProvider =
1556                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1557         routingProvider =
1558                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1559         l3ForwardingProvider =
1560                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1561         distributedArpService =
1562                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1563         nodeCacheManager =
1564                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1565         southbound =
1566                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1567         gatewayMacResolver =
1568                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1569         securityServicesManager =
1570                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1571
1572         initL3AdapterMembers();
1573     }
1574
1575     @Override
1576     public void setDependencies(Object impl) {
1577         if (impl instanceof INeutronNetworkCRUD) {
1578             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1579         } else if (impl instanceof INeutronPortCRUD) {
1580             neutronPortCache = (INeutronPortCRUD)impl;
1581         } else if (impl instanceof INeutronSubnetCRUD) {
1582             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1583         } else if (impl instanceof INeutronFloatingIPCRUD) {
1584             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1585         } else if (impl instanceof ArpProvider) {
1586             arpProvider = (ArpProvider)impl;
1587         } else if (impl instanceof InboundNatProvider) {
1588             inboundNatProvider = (InboundNatProvider)impl;
1589         } else if (impl instanceof OutboundNatProvider) {
1590             outboundNatProvider = (OutboundNatProvider)impl;
1591         } else if (impl instanceof RoutingProvider) {
1592             routingProvider = (RoutingProvider)impl;
1593         } else if (impl instanceof L3ForwardingProvider) {
1594             l3ForwardingProvider = (L3ForwardingProvider)impl;
1595         }else if (impl instanceof GatewayMacResolver) {
1596             gatewayMacResolver = (GatewayMacResolver)impl;
1597         }else if (impl instanceof IcmpEchoProvider) {
1598             icmpEchoProvider = (IcmpEchoProvider)impl;
1599         }
1600
1601         populateL3ForwardingCaches();
1602     }
1603 }