openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  15 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  16 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  17 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  18 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  19 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.IcmpEchoProvider;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  37 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  38 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  39 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  49 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  50 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  60 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  61 import org.osgi.framework.ServiceReference;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 import java.net.InetAddress;
  66 import java.net.UnknownHostException;
  67 import java.util.ArrayList;
  68 import java.util.Collection;
  69 import java.util.HashMap;
  70 import java.util.HashSet;
  71 import java.util.Iterator;
  72 import java.util.List;
  73 import java.util.Map;
  74 import java.util.Set;
  75
  76 /**
  77  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  78  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  79  * as well as the multi-tenant router forwarding provider.
  80  */
  81 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  82     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  83
  84     // The implementation for each of these services is resolved by the OSGi Service Manager
  85     private volatile ConfigurationService configurationService;
  86     private volatile TenantNetworkManager tenantNetworkManager;
  87     private volatile NodeCacheManager nodeCacheManager;
  88     private volatile INeutronNetworkCRUD neutronNetworkCache;
  89     private volatile INeutronSubnetCRUD neutronSubnetCache;
  90     private volatile INeutronPortCRUD neutronPortCache;
  91     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  92     private volatile L3ForwardingProvider l3ForwardingProvider;
  93     private volatile InboundNatProvider inboundNatProvider;
  94     private volatile OutboundNatProvider outboundNatProvider;
  95     private volatile ArpProvider arpProvider;
  96     private volatile RoutingProvider routingProvider;
  97     private volatile GatewayMacResolver gatewayMacResolver;
  98     private volatile SecurityServicesManager securityServicesManager;
  99     private volatile IcmpEchoProvider icmpEchoProvider;
 100
 101     private class FloatIpData {
 102         // br-int of node where floating ip is associated with tenant port
 103         private final Long dpid;
 104         // patch port in br-int used to reach br-ex
 105         private final Long ofPort;
 106         // segmentation id of the net where fixed ip is instantiated
 107         private final String segId;
 108         // mac address assigned to neutron port of floating ip
 109         private final String macAddress;
 110         private final String floatingIpAddress;
 111         // ip address given to tenant vm
 112         private final String fixedIpAddress;
 113         private final String neutronRouterMac;
 114
 115         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 116                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 117             this.dpid = dpid;
 118             this.ofPort = ofPort;
 119             this.segId = segId;
 120             this.macAddress = macAddress;
 121             this.floatingIpAddress = floatingIpAddress;
 122             this.fixedIpAddress = fixedIpAddress;
 123             this.neutronRouterMac = neutronRouterMac;
 124         }
 125     }
 126
 127     private Map<String, String> networkIdToRouterMacCache;
 128     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 129     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 130
 131     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 132     private Map<String, FloatIpData> floatIpDataMapCache;
 133
 134     private String externalRouterMac;
 135     private Boolean enabled = false;
 136     private Boolean isCachePopulationDone = false;
 137     private Map<String, NeutronPort> portCleanupCache;
 138     private Map<String, NeutronNetwork> networkCleanupCache;
 139
 140     private Southbound southbound;
 141     private DistributedArpService distributedArpService;
 142     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 143
 144     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 145     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 146     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 147     private static final String OWNER_FLOATING_IP = "network:floatingip";
 148     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 149
 150     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 151         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 152         this.neutronModelsDataStoreHelper = neutronHelper;
 153     }
 154
 155     private void initL3AdapterMembers() {
 156         Preconditions.checkNotNull(configurationService);
 157
 158         if (configurationService.isL3ForwardingEnabled()) {
 159             this.networkIdToRouterMacCache = new HashMap<>();
 160             this.networkIdToRouterIpListCache = new HashMap<>();
 161             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 162             this.neutronPortToDpIdCache = new HashMap<>();
 163             this.floatIpDataMapCache = new HashMap<>();
 164
 165             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 166             if (this.externalRouterMac == null) {
 167                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 168             }
 169             this.enabled = true;
 170             LOG.info("OVSDB L3 forwarding is enabled");
 171         } else {
 172             LOG.debug("OVSDB L3 forwarding is disabled");
 173         }
 174         this.portCleanupCache = new HashMap<>();
 175         this.networkCleanupCache = new HashMap<>();
 176     }
 177
 178     //
 179     // Callbacks from AbstractHandler
 180     //
 181     @Override
 182     public void processEvent(AbstractEvent abstractEvent) {
 183         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 184             LOG.error("Unable to process abstract event " + abstractEvent);
 185             return;
 186         }
 187         if (!this.enabled) {
 188             return;
 189         }
 190
 191         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 192         switch (ev.getAction()) {
 193             case UPDATE:
 194                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 195                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 196                 } else {
 197                     LOG.warn("Received update for an unexpected event " + ev);
 198                 }
 199                 break;
 200             case ADD:
 201                 // fall through...
 202                 // break;
 203             case DELETE:
 204                 // fall through...
 205                 // break;
 206             default:
 207                 LOG.warn("Unable to process event " + ev);
 208                 break;
 209         }
 210     }
 211
 212     //
 213     // Callbacks from GatewayMacResolverListener
 214     //
 215
 216     @Override
 217     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 218         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 219                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 220         if (!this.enabled) {
 221             return;
 222         }
 223
 224         if (macAddress == null || macAddress.getValue() == null) {
 225             // TODO: handle cases when mac is null
 226             return;
 227         }
 228
 229         //
 230         // Enqueue event so update is handled by adapter's thread
 231         //
 232         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 233     }
 234
 235     private void populateL3ForwardingCaches() {
 236         if (!this.enabled) {
 237             return;
 238         }
 239         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 240                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 241             return;
 242         }
 243         this.isCachePopulationDone = true;
 244         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 245         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 246         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 247         if(routers != null && routers.getRouter() != null && ports != null) {
 248             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 249             for( Router router : routers.getRouter()) {
 250                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 251                 if(!ports.getPort().isEmpty()) {
 252                     for( Port port : ports.getPort()) {
 253                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 254                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 255                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 256                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 257                                     , port.getMacAddress());
 258
 259                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 260                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 261                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 262                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 263                         }
 264                     }
 265                 }else {
 266                     LOG.warn("L3 Cache Population :Did not find any port information " +
 267                             "in config Data Store for router {}",router);
 268                 }
 269             }
 270         }
 271         LOG.debug("NetVirt L3 caches population is done");
 272     }
 273
 274     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 275         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 276             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 277             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 278             for(Node bridge : bridges) {
 279                 List<OvsdbTerminationPointAugmentation> interfaces
 280                         = southbound.extractTerminationPointAugmentations(bridge);
 281                 if(interfaces != null && !interfaces.isEmpty()) {
 282                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 283                             interfaces.size(), bridge.getNodeId());
 284                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 285                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 286                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 287                             Long dpId = getDpidForIntegrationBridge(bridge);
 288                             Uuid interfaceUuid = intf.getInterfaceUuid();
 289                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 290                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 291                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 292                             break;
 293                         }
 294                     }
 295                 }
 296             }
 297         }
 298         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 299     }
 300
 301     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 302         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 303         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 304         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 305             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 306                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 307                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 308                             "Fetching from data store.",neutronFloatingIP.getID());
 309                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 310                 }
 311             }
 312         }
 313         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 314         return floatIpDataMapCache.values();
 315     }
 316     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 317         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 318                 "floating id {} ",neutronFloatingId);
 319         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 320             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 321             if (neutronFloatingIP == null) {
 322                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 323                 return null;
 324             }
 325             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 326             NeutronPort neutronPortForFloatIp = null;
 327             for (NeutronPort neutronPort : neutronPorts) {
 328                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 329                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 330                     neutronPortForFloatIp = neutronPort;
 331                     break;
 332                 }
 333             }
 334
 335             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 336             if(neutronTenantPortUuid == null) {
 337                 return null;
 338             }
 339             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 340             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 341             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 342             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 343
 344             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 345             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 346                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 347             String providerSegmentationId = tenantNeutronNetwork != null ?
 348                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 349             String neutronRouterMac = tenantNeutronNetwork != null ?
 350                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 351
 352             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 353                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 354                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 355                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 356                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 357                                 "seg {} mac {} rtrMac {}",
 358                         fixedIpAddress,
 359                         floatingIpAddress,
 360                         neutronPortForFloatIp,
 361                         neutronTenantPortUuid,
 362                         providerSegmentationId,
 363                         floatingIpMac,
 364                         neutronRouterMac);
 365
 366                 return null;
 367             }
 368
 369             // get ofport for patch port in br-int
 370             final Long dpId = nodeIfPair.getLeft();
 371             final Long ofPort = findOFPortForExtPatch(dpId);
 372             if (ofPort == null) {
 373                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 374                                 "to connect floating ip to external bridge. dpid {}",
 375                         dpId);
 376                 return null;
 377             }
 378
 379             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 380                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 381             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 382
 383         }
 384         return floatIpDataMapCache.get(neutronFloatingId);
 385     }
 386     /**
 387      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 388      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 389      * subscribe so that the mac address associated to the gateway ip address is available.
 390      *
 391      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 392      */
 393     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 394         Preconditions.checkNotNull(externalRouterMacUpdate);
 395
 396         flushExistingIpRewrite();
 397         this.externalRouterMac = externalRouterMacUpdate;
 398         rebuildExistingIpRewrite();
 399     }
 400
 401     /**
 402      * Process the event.
 403      *
 404      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 405      * @param subnet An instance of NeutronSubnet object.
 406      */
 407     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 408         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 409         if (action == Action.ADD) {
 410             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 411         }
 412     }
 413
 414     /**
 415      * Process the port event as a router interface event.
 416      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 417      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 418      *
 419      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 420      * @param neutronPort An instance of NeutronPort object.
 421      */
 422     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 423         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 424
 425         if (action == Action.UPDATE) {
 426             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 427             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 428             this.processSecurityGroupUpdate(neutronPort);
 429         }
 430
 431         if (!this.enabled) {
 432             return;
 433         }
 434
 435         final boolean isDelete = action == Action.DELETE;
 436
 437         if (action == Action.DELETE) {
 438             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 439             this.cleanupFloatingIPRules(neutronPort);
 440         }
 441
 442         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 443             if (!isDelete) {
 444                 LOG.info("Port {} is network router gateway interface, "
 445                         + "triggering gateway resolution for the attached external network", neutronPort);
 446                 this.triggerGatewayMacResolver(neutronPort);
 447             }else{
 448                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 449                 if (null == externalNetwork) {
 450                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 451                 }
 452
 453                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 454                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 455                     // TODO support IPv6
 456                     if (externalSubnet != null &&
 457                             externalSubnet.getIpVersion() == 4) {
 458                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 459                     }
 460                 }
 461             }
 462         }
 463
 464         // Treat the port event as a router interface event if the port belongs to router. This is a
 465         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 466         //
 467         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 468             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 469
 470             if (neutronPort.getFixedIPs() != null) {
 471                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 472                     NeutronRouter_Interface neutronRouterInterface =
 473                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 474                     // id of router interface to be same as subnet
 475                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 476                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 477
 478                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 479                 }
 480             }
 481         } else {
 482             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 483             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 484             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 485             // there.
 486             //
 487             if (!isDelete && neutronPort.getFixedIPs() != null) {
 488                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 489                     NeutronRouter_Interface neutronRouterInterface =
 490                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 491                     if (neutronRouterInterface != null) {
 492                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 493                     }
 494                 }
 495             }
 496             this.updateL3ForNeutronPort(neutronPort, isDelete);
 497         }
 498     }
 499
 500     /**
 501      * Process the event.
 502      *
 503      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 504      * @param neutronRouter An instance of NeutronRouter object.
 505      */
 506     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 507         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 508     }
 509
 510     /**
 511      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 512      * delete the ports on the same subnet.
 513      *
 514      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 515      * @param neutronRouter An instance of NeutronRouter object.
 516      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 517      */
 518     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 519                                                   final NeutronRouter_Interface neutronRouterInterface,
 520                                                   Action action) {
 521         LOG.debug("Router interface {} got event {}. Subnet {}",
 522                      neutronRouterInterface.getPortUUID(),
 523                      action,
 524                      neutronRouterInterface.getSubnetUUID());
 525         if (!this.enabled) {
 526             return;
 527         }
 528
 529         final boolean isDelete = action == Action.DELETE;
 530
 531         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 532
 533         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 534         // see if they are affected by l3
 535         //
 536         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 537             boolean currPortShouldBeDeleted = false;
 538             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 539             if (isDelete) {
 540                 if (neutronPort.getFixedIPs() != null) {
 541                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 542                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 543                             currPortShouldBeDeleted = true;
 544                             break;
 545                         }
 546                     }
 547                 }
 548             }
 549             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 550         }
 551
 552     }
 553
 554     /**
 555      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 556      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 557      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 558      * way around) on OpenFlow Table 100.
 559      *
 560      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 561      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 562      */
 563     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 564                                              Action actionIn) {
 565         Preconditions.checkNotNull(neutronFloatingIP);
 566
 567         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 568                 neutronFloatingIP.getFixedIPAddress(),
 569                 neutronFloatingIP.getFloatingIPAddress(),
 570                 neutronFloatingIP.getFloatingNetworkUUID());
 571         if (!this.enabled) {
 572             return;
 573         }
 574
 575         Action action;
 576
 577         // Consider action to be delete if getFixedIPAddress is null
 578         //
 579         if (neutronFloatingIP.getFixedIPAddress() == null) {
 580             action = Action.DELETE;
 581         } else {
 582             action = actionIn;
 583         }
 584
 585         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 586
 587         if (action != Action.DELETE) {
 588             // must be first, as it updates floatIpDataMapCache
 589             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 590
 591             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 592             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 593         } else {
 594             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 595             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 596
 597             // must be last, as it updates floatIpDataMapCache
 598             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 599         }
 600     }
 601
 602     /**
 603      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 604      * ip, otherwise for newer one.
 605      */
 606     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 607         Preconditions.checkNotNull(neutronFloatingIP);
 608
 609         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 610         if (fid == null) {
 611             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 612                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 613             return;
 614         }
 615         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 616                                       action);
 617     }
 618
 619     /**
 620      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 621      * ip, otherwise for newer one.
 622      */
 623     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 624         Preconditions.checkNotNull(neutronFloatingIP);
 625
 626         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 627         if (fid == null) {
 628             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 629                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 630             return;
 631         }
 632         programOutboundIpRewriteStage1(fid, action);
 633     }
 634
 635     private void flushExistingIpRewrite() {
 636         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 637             programOutboundIpRewriteStage1(fid, Action.DELETE);
 638         }
 639     }
 640
 641     private void rebuildExistingIpRewrite() {
 642         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 643             programOutboundIpRewriteStage1(fid, Action.ADD);
 644         }
 645     }
 646
 647     /**
 648      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 649      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 650      * requests.
 651      */
 652     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 653         Preconditions.checkNotNull(neutronFloatingIP);
 654         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 655         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 656
 657         // find bridge Node where floating ip is configured by looking up cache for its port
 658         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 659         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 660         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 661         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 662         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 663         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 664
 665         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 666         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 667                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 668         final String providerSegmentationId = tenantNeutronNetwork != null ?
 669                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 670         final String neutronRouterMac = tenantNeutronNetwork != null ?
 671                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 672
 673         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 674                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 675                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 676                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 677             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 678                     fixedIpAddress,
 679                     floatingIpAddress,
 680                     neutronPortForFloatIp,
 681                     neutronTenantPortUuid,
 682                     providerSegmentationId,
 683                     floatingIpMac,
 684                     neutronRouterMac);
 685             return;
 686         }
 687
 688         // get ofport for patch port in br-int
 689         final Long dpId = nodeIfPair.getLeft();
 690         final Long ofPort = findOFPortForExtPatch(dpId);
 691         if (ofPort == null) {
 692             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 693                     dpId);
 694             return;
 695         }
 696
 697         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 698         //
 699         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 700                 Action.ADD)) {
 701             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 702                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 703             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 704             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 705                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 706                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 707         }
 708     }
 709
 710     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 711         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 712         if (floatIpData == null) {
 713             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 714             return;
 715         }
 716
 717         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 718                 floatIpData.floatingIpAddress, Action.DELETE)) {
 719             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 720             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 721                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 722         }
 723     }
 724
 725     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 726         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 727             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 728                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 729                 return neutronPort;
 730             }
 731         }
 732         return null;
 733     }
 734
 735     private Long findOFPortForExtPatch(Long dpId) {
 736         final String brInt = configurationService.getIntegrationBridgeName();
 737         final String brExt = configurationService.getExternalBridgeName();
 738         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 739
 740         Preconditions.checkNotNull(dpId);
 741         Preconditions.checkNotNull(portNameInt);
 742
 743         final long dpidPrimitive = dpId;
 744         for (Node node : nodeCacheManager.getBridgeNodes()) {
 745             if (dpidPrimitive == southbound.getDataPathId(node)) {
 746                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 747                         southbound.getTerminationPointOfBridge(node, portNameInt);
 748                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 749             }
 750         }
 751         return null;
 752     }
 753
 754     /**
 755      * Process the event.
 756      *
 757      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 758      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 759      */
 760     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 761         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 762         if (action == Action.UPDATE) {
 763             this.updateNetworkInCleanupCache(neutronNetwork);
 764         }
 765     }
 766
 767     //
 768     // Callbacks from OVSDB's southbound handler
 769     //
 770     /**
 771      * Process the event.
 772      *
 773      * @param bridgeNode An instance of Node object.
 774      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 775      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 776      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 777      * object.
 778      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 779      */
 780     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 781                                      final NeutronNetwork neutronNetwork, Action action) {
 782         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 783                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 784
 785         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 786         if (action != Action.DELETE && neutronPort != null) {
 787             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 788             storePortInCleanupCache(neutronPort);
 789         }
 790
 791         if (!this.enabled) {
 792             return;
 793         }
 794
 795         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 796         final Uuid interfaceUuid = intf.getInterfaceUuid();
 797
 798         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 799                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 800
 801         if (neutronPort != null) {
 802             final String neutronPortUuid = neutronPort.getPortUUID();
 803
 804             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 805                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 806             }
 807
 808             handleNeutronPortEvent(neutronPort, action);
 809         }
 810
 811         if (action == Action.DELETE && interfaceUuid != null) {
 812             handleInterfaceEventDelete(intf, dpId);
 813         }
 814     }
 815
 816     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 817         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 818         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 819                 neutronPortUuid, dpId, interfaceUuid.getValue());
 820     }
 821
 822     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 823         // Remove entry from neutronPortToDpIdCache based on interface uuid
 824         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 825             final String currPortUuid = entry.getKey();
 826             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 827                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 828                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 829                 neutronPortToDpIdCache.remove(currPortUuid);
 830                 break;
 831             }
 832         }
 833     }
 834
 835     //
 836     // Internal helpers
 837     //
 838     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 839
 840         final String networkUUID = neutronPort.getNetworkUUID();
 841         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 842
 843         if(!isDelete) {
 844             // If there is no router interface handling the networkUUID, we are done
 845             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 846                 return;
 847             }
 848
 849             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 850             // router interface are handled via handleNeutronRouterInterfaceEvent.
 851             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 852                 return;
 853             }
 854         }
 855
 856         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 857         final String providerSegmentationId = neutronNetwork != null ?
 858                                               neutronNetwork.getProviderSegmentationID() : null;
 859         final String tenantMac = neutronPort.getMacAddress();
 860
 861         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 862             tenantMac == null || tenantMac.isEmpty()) {
 863             // done: go no further w/out all the info needed...
 864             return;
 865         }
 866
 867         final Action action = isDelete ? Action.DELETE : Action.ADD;
 868         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 869         if (nodes.isEmpty()) {
 870             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 871         }
 872         for (Node node : nodes) {
 873             final Long dpid = getDpidForIntegrationBridge(node);
 874             if (dpid == null) {
 875                 continue;
 876             }
 877             if (neutronPort.getFixedIPs() == null) {
 878                 continue;
 879             }
 880             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 881                 final String tenantIpStr = neutronIP.getIpAddress();
 882                 if (tenantIpStr.isEmpty()) {
 883                     continue;
 884                 }
 885
 886                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 887                 // still needed when routing to subnets non-local to node (bug 2076).
 888                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 889             }
 890         }
 891     }
 892
 893     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 894         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 895         /**
 896          * Get updated data and original data for the the changed. Identify the security groups that got
 897          * added and removed and call the appropriate providers for updating the flows.
 898          */
 899         try {
 900             NeutronPort originalPort = neutronPort.getOriginalPort();
 901             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 902                                                                             neutronPort.getOriginalPort());
 903             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 904                                                                               neutronPort);
 905
 906             if (null != addedGroup && !addedGroup.isEmpty()) {
 907                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 908             }
 909             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 910                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 911             }
 912
 913         } catch (Exception e) {
 914             LOG.error("Exception in processSecurityGroupUpdate", e);
 915         }
 916     }
 917
 918     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 919         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 920         if (port1 == null) {
 921             return null;
 922         }
 923         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 924         if (port2 == null) {
 925             return list1;
 926         }
 927         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 928         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 929             NeutronSecurityGroup securityGroup1 = iterator.next();
 930             for (NeutronSecurityGroup securityGroup2 :list2) {
 931                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 932                     iterator.remove();
 933                 }
 934             }
 935         }
 936         return list1;
 937     }
 938
 939     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 940                                            String macAddress, String ipStr,
 941                                            Action actionForNode) {
 942         if (actionForNode == Action.DELETE) {
 943             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 944                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 945         }
 946         if (actionForNode == Action.ADD) {
 947             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 948                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 949         }
 950
 951         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 952                                                        macAddress, ipStr, actionForNode);
 953     }
 954
 955     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 956                                              String macAddress,
 957                                              String address,
 958                                              Action actionForNode) {
 959         Status status;
 960         try {
 961             InetAddress inetAddress = InetAddress.getByName(address);
 962             status = l3ForwardingProvider == null ?
 963                      new Status(StatusCode.SUCCESS) :
 964                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 965                                                                       inetAddress, macAddress, actionForNode);
 966         } catch (UnknownHostException e) {
 967             status = new Status(StatusCode.BADREQUEST);
 968         }
 969
 970         if (status.isSuccess()) {
 971             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 972                          l3ForwardingProvider == null ? "skipped" : "programmed",
 973                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 974         } else {
 975             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 976                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 977         }
 978         return status;
 979     }
 980
 981     // --
 982
 983     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 984                                                        Boolean isDelete) {
 985         Preconditions.checkNotNull(destNeutronRouterInterface);
 986
 987         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 988         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 989         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 990         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 991         final NeutronNetwork neutronNetwork = subnet != null ?
 992                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 993         final String destinationSegmentationId = neutronNetwork != null ?
 994                                                  neutronNetwork.getProviderSegmentationID() : null;
 995         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 996         final String cidr = subnet != null ? subnet.getCidr() : null;
 997         final int mask = getMaskLenFromCidr(cidr);
 998
 999         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1000                      destNeutronRouterInterface, isDelete);
1001
1002         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1003         // the local cache
1004         if (neutronNetwork != null) {
1005             if (macAddress == null || macAddress.isEmpty()) {
1006                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1007             }
1008             if (ipList == null || ipList.isEmpty()) {
1009                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1010             }
1011         }
1012
1013         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1014             cidr == null || cidr.isEmpty() ||
1015             macAddress == null || macAddress.isEmpty() ||
1016             ipList == null || ipList.isEmpty()) {
1017             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1018                          destinationSegmentationId, cidr, macAddress, ipList);
1019             // done: go no further w/out all the info needed...
1020             return;
1021         }
1022
1023         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1024
1025         // Keep cache for finding router's mac from network uuid -- add
1026         //
1027         if (! isDelete) {
1028             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1029             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1030             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1031         }
1032
1033         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1034         if (nodes.isEmpty()) {
1035             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1036         }
1037         for (Node node : nodes) {
1038             final Long dpid = getDpidForIntegrationBridge(node);
1039             if (dpid == null) {
1040                 continue;
1041             }
1042
1043             for (Neutron_IPs neutronIP : ipList) {
1044                 final String ipStr = neutronIP.getIpAddress();
1045                 if (ipStr.isEmpty()) {
1046                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1047                             node.getNodeId().getValue(), ipStr);
1048                     continue;
1049                 }
1050
1051                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1052                 //
1053                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1054                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1055                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1056                                                               neutronNetwork, destinationSegmentationId,
1057                                                               macAddress, ipStr, mask, actionForNode,
1058                                                               true /*isReflexsive*/);
1059                 }
1060
1061                 if (! isExternal) {
1062                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1063                             actionForNode);
1064                 }
1065                 // Enable ARP responder by default, because router interface needs to be responded always.
1066                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1067                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1068             }
1069
1070             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1071             // for the external neutron networks.
1072             //
1073             {
1074                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1075                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1076             }
1077         }
1078
1079         if (isDelete) {
1080             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1081             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1082             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1083         }
1084     }
1085
1086     private void programFlowForNetworkFromExternal(final Node node,
1087                                                    final Long dpid,
1088                                                    final String destinationSegmentationId,
1089                                                    final String dstMacAddress,
1090                                                    final String destIpStr,
1091                                                    final int destMask,
1092                                                    final Action actionForNode) {
1093         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1094                 dstMacAddress, destIpStr, destMask, actionForNode);
1095     }
1096
1097     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1098                                                            final Long dpid,
1099                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1100                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1101                                                            final NeutronNetwork dstNeutronNetwork,
1102                                                            final String destinationSegmentationId,
1103                                                            final String dstMacAddress,
1104                                                            final String destIpStr,
1105                                                            final int destMask,
1106                                                            final Action actionForNode,
1107                                                            Boolean isReflexsive) {
1108         Preconditions.checkNotNull(srcNeutronRouterInterface);
1109         Preconditions.checkNotNull(dstNeutronRouterInterface);
1110
1111         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1112         if (sourceSubnetId == null) {
1113             LOG.error("Could not get provider Subnet ID from router interface {}",
1114                          srcNeutronRouterInterface.getID());
1115             return;
1116         }
1117
1118         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1119         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1120         if (sourceNetworkId == null) {
1121             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1122             return;
1123         }
1124
1125         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1126         if (sourceNetwork == null) {
1127             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1128             return;
1129         }
1130
1131         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1132             // Isolate subnets from different tenants within the same router
1133             return;
1134         }
1135         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1136         if (sourceSegmentationId == null) {
1137             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1138             return;
1139         }
1140         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1141             // Skip 'self'
1142             return;
1143         }
1144
1145         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1146                                      dstMacAddress, destIpStr, destMask, actionForNode);
1147
1148         // Flip roles src->dst; dst->src
1149         if (isReflexsive) {
1150             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1151             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1152             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1153             final String cidr2 = sourceSubnet.getCidr();
1154             final int mask2 = getMaskLenFromCidr(cidr2);
1155
1156             if (cidr2 == null || cidr2.isEmpty() ||
1157                 macAddress2 == null || macAddress2.isEmpty() ||
1158                 ipList2 == null || ipList2.isEmpty()) {
1159                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1160                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1161                 // done: go no further w/out all the info needed...
1162                 return;
1163             }
1164
1165             for (Neutron_IPs neutronIP2 : ipList2) {
1166                 final String ipStr2 = neutronIP2.getIpAddress();
1167                 if (ipStr2.isEmpty()) {
1168                     continue;
1169                 }
1170                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1171                                                           srcNeutronRouterInterface,
1172                                                           sourceNetwork, sourceSegmentationId,
1173                                                           macAddress2, ipStr2, mask2, actionForNode,
1174                                                           false /*isReflexsive*/);
1175             }
1176         }
1177     }
1178
1179     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1180                                               String destinationSegmentationId,
1181                                               String macAddress, String ipStr, int mask,
1182                                               Action actionForNode) {
1183         if (actionForNode == Action.DELETE) {
1184             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1185                          " action {}",
1186                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1187                          macAddress, ipStr, mask, actionForNode);
1188         }
1189         if (actionForNode == Action.ADD) {
1190             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1191                          " action {}",
1192                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1193                          macAddress, ipStr, mask, actionForNode);
1194         }
1195
1196         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1197                                                           macAddress, ipStr, mask, actionForNode);
1198     }
1199
1200     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1201                                                 String destinationSegmentationId,
1202                                                 String macAddress,
1203                                                 String address, int mask,
1204                                                 Action actionForNode) {
1205         Status status;
1206         try {
1207             InetAddress inetAddress = InetAddress.getByName(address);
1208             status = routingProvider == null ?
1209                      new Status(StatusCode.SUCCESS) :
1210                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1211                                                             macAddress, inetAddress, mask, actionForNode);
1212         } catch (UnknownHostException e) {
1213             status = new Status(StatusCode.BADREQUEST);
1214         }
1215
1216         if (status.isSuccess()) {
1217             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1218                          routingProvider == null ? "skipped" : "programmed",
1219                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1220                          actionForNode);
1221         } else {
1222             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1223                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1224                          actionForNode, status);
1225         }
1226         return status;
1227     }
1228
1229     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1230                                            String macAddress, String ipStr,
1231                                            Action action) {
1232         if (action == Action.DELETE ) {
1233             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1234                     dpid, segOrOfPort, macAddress, ipStr, action);
1235         }
1236         if (action == Action.ADD) {
1237             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1238                     dpid, segOrOfPort, macAddress, ipStr, action);
1239         }
1240
1241         Status status = new Status(StatusCode.UNSUPPORTED);
1242         if (icmpEchoProvider != null){
1243             try {
1244                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1245                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1246                                                 macAddress, inetAddress, action);
1247             } catch (UnknownHostException e) {
1248                 status = new Status(StatusCode.BADREQUEST);
1249             }
1250         }
1251
1252         if (status.isSuccess()) {
1253             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1254                     icmpEchoProvider == null ? "skipped" : "programmed",
1255                     macAddress, ipStr, dpid, segOrOfPort, action);
1256         } else {
1257             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1258                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1259         }
1260
1261         return status.isSuccess();
1262     }
1263
1264     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1265                                                   String matchAddress, String rewriteAddress,
1266                                                   Action action) {
1267         if (action == Action.DELETE ) {
1268             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1269                     " action {}",
1270                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1271         }
1272         if (action == Action.ADD ) {
1273             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1274                     " action {}",
1275                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1276         }
1277
1278         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1279                 rewriteAddress, action);
1280         return status.isSuccess();
1281     }
1282
1283     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1284                                                  String matchAddress, String rewriteAddress,
1285                                                  Action action) {
1286         Status status;
1287         try {
1288             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1289             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1290             status = inboundNatProvider == null ?
1291                     new Status(StatusCode.SUCCESS) :
1292                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1293                             inetMatchAddress, inetRewriteAddress,
1294                             action);
1295         } catch (UnknownHostException e) {
1296             status = new Status(StatusCode.BADREQUEST);
1297         }
1298
1299         if (status.isSuccess()) {
1300             final boolean isSkipped = inboundNatProvider == null;
1301             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1302                     isSkipped ? "skipped" : "programmed",
1303                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1304         } else {
1305             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1306                          " status:{}",
1307                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1308                     status);
1309         }
1310         return status;
1311     }
1312
1313     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1314                                                  Action actionForRewriteExclusion) {
1315         if (actionForRewriteExclusion == Action.DELETE ) {
1316             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1317                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1318         }
1319         if (actionForRewriteExclusion == Action.ADD) {
1320             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1321                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1322         }
1323
1324         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1325     }
1326
1327     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1328                                                    Action actionForNode) {
1329         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1330                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1331
1332         if (status.isSuccess()) {
1333             final boolean isSkipped = outboundNatProvider == null;
1334             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1335                          isSkipped ? "skipped" : "programmed",
1336                          cidr, node.getNodeId().getValue(), actionForNode);
1337         } else {
1338             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1339                          cidr, node.getNodeId().getValue(), actionForNode, status);
1340         }
1341         return status;
1342     }
1343
1344     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1345
1346         if (action == Action.DELETE) {
1347             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1348                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1349         }
1350         if (action == Action.ADD) {
1351             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1352                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1353         }
1354
1355         this.programOutboundIpRewriteStage2(fid, action);
1356     }
1357
1358     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1359         Status status;
1360         try {
1361             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1362             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1363             status = outboundNatProvider == null ?
1364                     new Status(StatusCode.SUCCESS) :
1365                     outboundNatProvider.programIpRewriteRule(
1366                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1367                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1368         } catch (UnknownHostException e) {
1369             status = new Status(StatusCode.BADREQUEST);
1370         }
1371
1372         if (status.isSuccess()) {
1373             final boolean isSkipped = outboundNatProvider == null;
1374             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1375                             " action {}",
1376                          isSkipped ? "skipped" : "programmed",
1377                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1378         } else {
1379             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1380                          " action {} status:{}",
1381                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1382         }
1383         return status;
1384     }
1385
1386     private int getMaskLenFromCidr(String cidr) {
1387         if (cidr == null) {
1388             return 0;
1389         }
1390         String[] splits = cidr.split("/");
1391         if (splits.length != 2) {
1392             return 0;
1393         }
1394
1395         int result;
1396         try {
1397             result = Integer.parseInt(splits[1].trim());
1398         } catch (NumberFormatException nfe) {
1399             result = 0;
1400         }
1401         return result;
1402     }
1403
1404     private Long getDpidForIntegrationBridge(Node node) {
1405         // Check if node is integration bridge; and only then return its dpid
1406         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1407             return southbound.getDataPathId(node);
1408         }
1409         return null;
1410     }
1411
1412     private Long getDpidForExternalBridge(Node node) {
1413         // Check if node is external bridge; and only then return its dpid
1414         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1415             return southbound.getDataPathId(node);
1416         }
1417         return null;
1418     }
1419
1420     private Node getExternalBridgeNode(){
1421         //Pickup the first node that has external bridge (br-ex).
1422         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1423         //the external network is reachable from every br-ex
1424         // TODO: Consider other deployment scenario, and thing of better solution.
1425         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1426         for(Node node : allBridges){
1427             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1428                 return node;
1429             }
1430         }
1431         return null;
1432     }
1433
1434     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1435         if (gatewayPort.getFixedIPs() == null) {
1436             return null;
1437         }
1438         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1439             String subnetUUID = neutronIPs.getSubnetUUID();
1440             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1441             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1442                 return extSubnet;
1443             }
1444             if (extSubnet == null) {
1445                 // TODO: when subnet is created, try again.
1446                 LOG.debug("subnet {} in not found", subnetUUID);
1447              }
1448         }
1449         return null;
1450     }
1451
1452     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1453
1454         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1455         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1456             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1457                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1458                     handleNeutronFloatingIPEvent(neutronFloatingIP, Action.DELETE);
1459                 }
1460             }
1461         }
1462     }
1463
1464
1465     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1466
1467         Preconditions.checkNotNull(gatewayPort);
1468         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1469
1470         if(externalNetwork != null){
1471             if(externalNetwork.isRouterExternal()){
1472                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1473
1474                 // TODO: address IPv6 case.
1475                 if (externalSubnet != null &&
1476                         externalSubnet.getIpVersion() == 4 &&
1477                         gatewayPort.getFixedIPs() != null) {
1478                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1479
1480                     gatewayMacResolver.resolveMacAddress(
1481                             this, /* gatewayMacResolverListener */
1482                             null, /* externalNetworkBridgeDpid */
1483                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1484                             new Ipv4Address(externalSubnet.getGatewayIP()),
1485                             new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1486                             new MacAddress(gatewayPort.getMacAddress()),
1487                             true /* periodicRefresh */);
1488                 } else {
1489                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1490                 }
1491             }
1492         }else{
1493             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1494         }
1495     }
1496
1497
1498     private void storePortInCleanupCache(NeutronPort port) {
1499         this.portCleanupCache.put(port.getPortUUID(),port);
1500     }
1501
1502
1503     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1504         removePortFromCleanupCache(originalPort);
1505         storePortInCleanupCache(updatedPort);
1506     }
1507
1508     public void removePortFromCleanupCache(NeutronPort port) {
1509         if(port != null) {
1510             this.portCleanupCache.remove(port.getPortUUID());
1511         }
1512     }
1513
1514     public Map<String, NeutronPort> getPortCleanupCache() {
1515         return this.portCleanupCache;
1516     }
1517
1518     public NeutronPort getPortFromCleanupCache(String portid) {
1519         for (String neutronPortUuid : this.portCleanupCache.keySet()) {
1520             if (neutronPortUuid.equals(portid)) {
1521                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1522                 return this.portCleanupCache.get(neutronPortUuid);
1523             }
1524         }
1525         return null;
1526     }
1527
1528     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1529         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1530     }
1531
1532
1533     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1534         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1535             if (neutronNetworkUuid.equals(network.getNetworkUUID())) {
1536                 this.networkCleanupCache.remove(neutronNetworkUuid);
1537             }
1538         }
1539         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1540     }
1541
1542     public void removeNetworkFromCleanupCache(String networkid) {
1543         NeutronNetwork network = null;
1544         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1545             if (neutronNetworkUuid.equals(networkid)) {
1546                 network = networkCleanupCache.get(neutronNetworkUuid);
1547                 break;
1548             }
1549         }
1550         if (network != null) {
1551             for (String neutronPortUuid:this.portCleanupCache.keySet()) {
1552                 if (this.portCleanupCache.get(neutronPortUuid).getNetworkUUID().equals(network.getNetworkUUID())) {
1553                     LOG.info("This network is used by another port", network);
1554                     return;
1555                 }
1556             }
1557             this.networkCleanupCache.remove(network.getNetworkUUID());
1558         }
1559     }
1560
1561     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1562         return this.networkCleanupCache;
1563     }
1564
1565     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1566         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1567             if (neutronNetworkUuid.equals(networkid)) {
1568                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", networkid);
1569                 return networkCleanupCache.get(neutronNetworkUuid);
1570             }
1571         }
1572         return null;
1573     }
1574     /**
1575      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1576      *
1577      * @param ofPort the OF port number
1578      * @return the string with encoded OF port (example format "OFPort|999")
1579      */
1580     public static String encodeExcplicitOFPort(Long ofPort) {
1581         return "OFPort|" + ofPort.toString();
1582     }
1583     private void initNetworkCleanUpCache() {
1584         if (this.neutronNetworkCache != null) {
1585             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1586                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1587             }
1588         }
1589     }
1590     private void initPortCleanUpCache() {
1591         if (this.neutronPortCache != null) {
1592             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1593                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1594             }
1595         }
1596     }
1597     @Override
1598     public void setDependencies(ServiceReference serviceReference) {
1599         eventDispatcher =
1600                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1601         eventDispatcher.eventHandlerAdded(serviceReference, this);
1602         tenantNetworkManager =
1603                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1604         configurationService =
1605                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1606         arpProvider =
1607                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1608         inboundNatProvider =
1609                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1610         outboundNatProvider =
1611                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1612         routingProvider =
1613                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1614         l3ForwardingProvider =
1615                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1616         distributedArpService =
1617                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1618         nodeCacheManager =
1619                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1620         southbound =
1621                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1622         gatewayMacResolver =
1623                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1624         securityServicesManager =
1625                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1626
1627         initL3AdapterMembers();
1628     }
1629
1630     @Override
1631     public void setDependencies(Object impl) {
1632         if (impl instanceof INeutronNetworkCRUD) {
1633             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1634             initNetworkCleanUpCache();
1635         } else if (impl instanceof INeutronPortCRUD) {
1636             neutronPortCache = (INeutronPortCRUD)impl;
1637             initPortCleanUpCache();
1638         } else if (impl instanceof INeutronSubnetCRUD) {
1639             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1640         } else if (impl instanceof INeutronFloatingIPCRUD) {
1641             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1642         } else if (impl instanceof ArpProvider) {
1643             arpProvider = (ArpProvider)impl;
1644         } else if (impl instanceof InboundNatProvider) {
1645             inboundNatProvider = (InboundNatProvider)impl;
1646         } else if (impl instanceof OutboundNatProvider) {
1647             outboundNatProvider = (OutboundNatProvider)impl;
1648         } else if (impl instanceof RoutingProvider) {
1649             routingProvider = (RoutingProvider)impl;
1650         } else if (impl instanceof L3ForwardingProvider) {
1651             l3ForwardingProvider = (L3ForwardingProvider)impl;
1652         }else if (impl instanceof GatewayMacResolver) {
1653             gatewayMacResolver = (GatewayMacResolver)impl;
1654         }else if (impl instanceof IcmpEchoProvider) {
1655             icmpEchoProvider = (IcmpEchoProvider)impl;
1656         }
1657
1658         populateL3ForwardingCaches();
1659     }
1660 }