openstack/net-virt/src/main/java/org/opendaylight/netvirt/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.openstack.netvirt.impl;
  10
  11 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.ADD;
  12 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.DELETE;
  13 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.UPDATE;
  14
  15 import com.google.common.base.Preconditions;
  16 import org.apache.commons.lang3.tuple.ImmutablePair;
  17 import org.apache.commons.lang3.tuple.Pair;
  18 import org.opendaylight.netvirt.openstack.netvirt.AbstractEvent;
  19 import org.opendaylight.netvirt.openstack.netvirt.AbstractHandler;
  20 import org.opendaylight.netvirt.openstack.netvirt.ConfigInterface;
  21 import org.opendaylight.netvirt.openstack.netvirt.NeutronL3AdapterEvent;
  22 import org.opendaylight.netvirt.openstack.netvirt.api.Action;
  23 import org.opendaylight.netvirt.openstack.netvirt.api.ArpProvider;
  24 import org.opendaylight.netvirt.openstack.netvirt.api.ConfigurationService;
  25 import org.opendaylight.netvirt.openstack.netvirt.api.Constants;
  26 import org.opendaylight.netvirt.openstack.netvirt.api.EventDispatcher;
  27 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolver;
  28 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolverListener;
  29 import org.opendaylight.netvirt.openstack.netvirt.api.IcmpEchoProvider;
  30 import org.opendaylight.netvirt.openstack.netvirt.api.InboundNatProvider;
  31 import org.opendaylight.netvirt.openstack.netvirt.api.L3ForwardingProvider;
  32 import org.opendaylight.netvirt.openstack.netvirt.api.NodeCacheManager;
  33 import org.opendaylight.netvirt.openstack.netvirt.api.OutboundNatProvider;
  34 import org.opendaylight.netvirt.openstack.netvirt.api.RoutingProvider;
  35 import org.opendaylight.netvirt.openstack.netvirt.api.SecurityServicesManager;
  36 import org.opendaylight.netvirt.openstack.netvirt.api.Southbound;
  37 import org.opendaylight.netvirt.openstack.netvirt.api.Status;
  38 import org.opendaylight.netvirt.openstack.netvirt.api.StatusCode;
  39 import org.opendaylight.netvirt.openstack.netvirt.api.TenantNetworkManager;
  40 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronFloatingIP;
  41 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronNetwork;
  42 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter;
  43 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter_Interface;
  44 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSecurityGroup;
  45 import org.opendaylight.netvirt.openstack.netvirt.translator.Neutron_IPs;
  46 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  47 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  48 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronPortCRUD;
  49 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronPort;
  50 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSubnet;
  51 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  52 import org.opendaylight.netvirt.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  53 import org.opendaylight.netvirt.utils.neutron.utils.NeutronModelsDataStoreHelper;
  54 import org.opendaylight.netvirt.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  64 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  65 import org.osgi.framework.ServiceReference;
  66 import org.slf4j.Logger;
  67 import org.slf4j.LoggerFactory;
  68
  69 import java.net.Inet4Address;
  70 import java.net.InetAddress;
  71 import java.net.UnknownHostException;
  72 import java.util.ArrayList;
  73 import java.util.Collection;
  74 import java.util.HashMap;
  75 import java.util.Iterator;
  76 import java.util.List;
  77 import java.util.Map;
  78
  79 /**
  80  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  81  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  82  * as well as the multi-tenant router forwarding provider.
  83  */
  84 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  85     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  86
  87     // The implementation for each of these services is resolved by the OSGi Service Manager
  88     private volatile ConfigurationService configurationService;
  89     private volatile TenantNetworkManager tenantNetworkManager;
  90     private volatile NodeCacheManager nodeCacheManager;
  91     private volatile INeutronNetworkCRUD neutronNetworkCache;
  92     private volatile INeutronSubnetCRUD neutronSubnetCache;
  93     private volatile INeutronPortCRUD neutronPortCache;
  94     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  95     private volatile L3ForwardingProvider l3ForwardingProvider;
  96     private volatile InboundNatProvider inboundNatProvider;
  97     private volatile OutboundNatProvider outboundNatProvider;
  98     private volatile ArpProvider arpProvider;
  99     private volatile RoutingProvider routingProvider;
 100     private volatile GatewayMacResolver gatewayMacResolver;
 101     private volatile SecurityServicesManager securityServicesManager;
 102     private volatile IcmpEchoProvider icmpEchoProvider;
 103
 104     private class FloatIpData {
 105         // br-int of node where floating ip is associated with tenant port
 106         private final Long dpid;
 107         // patch port in br-int used to reach br-ex
 108         private final Long ofPort;
 109         // segmentation id of the net where fixed ip is instantiated
 110         private final String segId;
 111         // mac address assigned to neutron port of floating ip
 112         private final String macAddress;
 113         private final String floatingIpAddress;
 114         // ip address given to tenant vm
 115         private final String fixedIpAddress;
 116         private final String neutronRouterMac;
 117
 118         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 119                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 120             this.dpid = dpid;
 121             this.ofPort = ofPort;
 122             this.segId = segId;
 123             this.macAddress = macAddress;
 124             this.floatingIpAddress = floatingIpAddress;
 125             this.fixedIpAddress = fixedIpAddress;
 126             this.neutronRouterMac = neutronRouterMac;
 127         }
 128     }
 129
 130     private Map<String, String> networkIdToRouterMacCache;
 131     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 132     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 133
 134     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 135     private Map<String, FloatIpData> floatIpDataMapCache;
 136
 137     private String externalRouterMac;
 138     private Boolean enabled = false;
 139     private Boolean isCachePopulationDone = false;
 140     private Map<String, NeutronPort> portCleanupCache;
 141     private Map<String, NeutronNetwork> networkCleanupCache;
 142
 143     private Southbound southbound;
 144     private DistributedArpService distributedArpService;
 145     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 146
 147     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 148     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 149     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 150     private static final String OWNER_FLOATING_IP = "network:floatingip";
 151     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 152
 153     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 154         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 155         this.neutronModelsDataStoreHelper = neutronHelper;
 156     }
 157
 158     private void initL3AdapterMembers() {
 159         Preconditions.checkNotNull(configurationService);
 160
 161         if (configurationService.isL3ForwardingEnabled()) {
 162             this.networkIdToRouterMacCache = new HashMap<>();
 163             this.networkIdToRouterIpListCache = new HashMap<>();
 164             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 165             this.neutronPortToDpIdCache = new HashMap<>();
 166             this.floatIpDataMapCache = new HashMap<>();
 167
 168             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 169             if (this.externalRouterMac == null) {
 170                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 171             }
 172             this.enabled = true;
 173             LOG.info("OVSDB L3 forwarding is enabled");
 174         } else {
 175             LOG.debug("OVSDB L3 forwarding is disabled");
 176         }
 177         this.portCleanupCache = new HashMap<>();
 178         this.networkCleanupCache = new HashMap<>();
 179     }
 180
 181     //
 182     // Callbacks from AbstractHandler
 183     //
 184     @Override
 185     public void processEvent(AbstractEvent abstractEvent) {
 186         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 187             LOG.error("Unable to process abstract event " + abstractEvent);
 188             return;
 189         }
 190         if (!this.enabled) {
 191             return;
 192         }
 193
 194         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 195         switch (ev.getAction()) {
 196             case UPDATE:
 197                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 198                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 199                 } else {
 200                     LOG.warn("Received update for an unexpected event " + ev);
 201                 }
 202                 break;
 203             case ADD:
 204                 // fall through...
 205                 // break;
 206             case DELETE:
 207                 // fall through...
 208                 // break;
 209             default:
 210                 LOG.warn("Unable to process event " + ev);
 211                 break;
 212         }
 213     }
 214
 215     //
 216     // Callbacks from GatewayMacResolverListener
 217     //
 218
 219     @Override
 220     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 221         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 222                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 223         if (!this.enabled) {
 224             return;
 225         }
 226
 227         if (macAddress == null || macAddress.getValue() == null) {
 228             // TODO: handle cases when mac is null
 229             return;
 230         }
 231
 232         //
 233         // Enqueue event so update is handled by adapter's thread
 234         //
 235         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 236     }
 237
 238     private void populateL3ForwardingCaches() {
 239         if (!this.enabled) {
 240             return;
 241         }
 242         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 243                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 244             return;
 245         }
 246         this.isCachePopulationDone = true;
 247         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 248         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 249         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 250         if(routers != null && routers.getRouter() != null && ports != null) {
 251             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 252             for( Router router : routers.getRouter()) {
 253                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 254                 if(!ports.getPort().isEmpty()) {
 255                     for( Port port : ports.getPort()) {
 256                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 257                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 258                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 259                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 260                                     , port.getMacAddress());
 261
 262                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 263                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 264                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 265                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 266                         }
 267                     }
 268                 }else {
 269                     LOG.warn("L3 Cache Population :Did not find any port information " +
 270                             "in config Data Store for router {}",router);
 271                 }
 272             }
 273         }
 274         LOG.debug("NetVirt L3 caches population is done");
 275     }
 276
 277     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 278         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 279             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 280             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 281             for(Node bridge : bridges) {
 282                 List<OvsdbTerminationPointAugmentation> interfaces
 283                         = southbound.extractTerminationPointAugmentations(bridge);
 284                 if(interfaces != null && !interfaces.isEmpty()) {
 285                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 286                             interfaces.size(), bridge.getNodeId());
 287                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 288                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 289                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 290                             Long dpId = getDpidForIntegrationBridge(bridge);
 291                             Uuid interfaceUuid = intf.getInterfaceUuid();
 292                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 293                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 294                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 295                             break;
 296                         }
 297                     }
 298                 }
 299             }
 300         }
 301         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 302     }
 303
 304     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 305         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 306         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 307         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 308             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 309                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 310                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 311                             "Fetching from data store.",neutronFloatingIP.getID());
 312                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 313                 }
 314             }
 315         }
 316         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 317         return floatIpDataMapCache.values();
 318     }
 319     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 320         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 321                 "floating id {} ",neutronFloatingId);
 322         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 323             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 324             if (neutronFloatingIP == null) {
 325                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 326                 return null;
 327             }
 328             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 329             NeutronPort neutronPortForFloatIp = null;
 330             for (NeutronPort neutronPort : neutronPorts) {
 331                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 332                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 333                     neutronPortForFloatIp = neutronPort;
 334                     break;
 335                 }
 336             }
 337
 338             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 339             if(neutronTenantPortUuid == null) {
 340                 return null;
 341             }
 342             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 343             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 344             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 345             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 346
 347             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 348             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 349                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 350             String providerSegmentationId = tenantNeutronNetwork != null ?
 351                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 352             String neutronRouterMac = tenantNeutronNetwork != null ?
 353                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 354
 355             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 356                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 357                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 358                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 359                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 360                                 "seg {} mac {} rtrMac {}",
 361                         fixedIpAddress,
 362                         floatingIpAddress,
 363                         neutronPortForFloatIp,
 364                         neutronTenantPortUuid,
 365                         providerSegmentationId,
 366                         floatingIpMac,
 367                         neutronRouterMac);
 368
 369                 return null;
 370             }
 371
 372             // get ofport for patch port in br-int
 373             final Long dpId = nodeIfPair.getLeft();
 374             final Long ofPort = findOFPortForExtPatch(dpId);
 375             if (ofPort == null) {
 376                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 377                                 "to connect floating ip to external bridge. dpid {}",
 378                         dpId);
 379                 return null;
 380             }
 381
 382             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 383                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 384             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 385
 386         }
 387         return floatIpDataMapCache.get(neutronFloatingId);
 388     }
 389     /**
 390      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 391      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 392      * subscribe so that the mac address associated to the gateway ip address is available.
 393      *
 394      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 395      */
 396     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 397         Preconditions.checkNotNull(externalRouterMacUpdate);
 398
 399         flushExistingIpRewrite();
 400         this.externalRouterMac = externalRouterMacUpdate;
 401         rebuildExistingIpRewrite();
 402     }
 403
 404     /**
 405      * Process the event.
 406      *
 407      * @param action the {@link Action} action to be handled.
 408      * @param subnet An instance of NeutronSubnet object.
 409      */
 410     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 411         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 412         if (action == ADD) {
 413             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 414         }
 415     }
 416
 417     /**
 418      * Process the port event as a router interface event.
 419      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 420      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 421      *
 422      * @param action the {@link Action} action to be handled.
 423      * @param neutronPort An instance of NeutronPort object.
 424      */
 425     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 426         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 427
 428         if (action == UPDATE) {
 429             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 430             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 431             if (neutronPort.getPortSecurityEnabled()) {
 432                 this.processSecurityGroupUpdate(neutronPort);
 433             }
 434         }
 435
 436         if (!this.enabled) {
 437             return;
 438         }
 439
 440         final boolean isDelete = action == DELETE;
 441
 442         if (action == DELETE) {
 443             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 444             this.cleanupFloatingIPRules(neutronPort);
 445         }
 446         else if (action == UPDATE){
 447             // Bug 5353: VM restart cause floatingIp flows to be removed
 448             this.updateFloatingIPRules(neutronPort);
 449         }
 450
 451         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 452             if (!isDelete) {
 453                 LOG.info("Port {} is network router gateway interface, "
 454                         + "triggering gateway resolution for the attached external network", neutronPort);
 455                 this.triggerGatewayMacResolver(neutronPort);
 456             }else{
 457                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 458                 if (null == externalNetwork) {
 459                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 460                 }
 461
 462                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 463                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 464                     // TODO support IPv6
 465                     if (externalSubnet != null &&
 466                             externalSubnet.getIpVersion() == 4) {
 467                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 468                     }
 469                 }
 470             }
 471         }
 472
 473         // Treat the port event as a router interface event if the port belongs to router. This is a
 474         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 475         //
 476         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 477             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 478
 479             if (neutronPort.getFixedIPs() != null) {
 480                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 481                     NeutronRouter_Interface neutronRouterInterface =
 482                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 483                     // id of router interface to be same as subnet
 484                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 485                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 486
 487                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 488                 }
 489             }
 490         } else {
 491             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 492             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 493             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 494             // there.
 495             //
 496             if (!isDelete && neutronPort.getFixedIPs() != null) {
 497                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 498                     NeutronRouter_Interface neutronRouterInterface =
 499                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 500                     if (neutronRouterInterface != null) {
 501                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 502                     }
 503                 }
 504             }
 505             this.updateL3ForNeutronPort(neutronPort, isDelete);
 506         }
 507     }
 508
 509     /**
 510      * Process the event.
 511      *
 512      * @param action the {@link Action} action to be handled.
 513      * @param neutronRouter An instance of NeutronRouter object.
 514      */
 515     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 516         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 517     }
 518
 519     /**
 520      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 521      * delete the ports on the same subnet.
 522      *
 523      * @param action the {@link Action} action to be handled.
 524      * @param neutronRouter An instance of NeutronRouter object.
 525      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 526      */
 527     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 528                                                   final NeutronRouter_Interface neutronRouterInterface,
 529                                                   Action action) {
 530         LOG.debug("Router interface {} got event {}. Subnet {}",
 531                      neutronRouterInterface.getPortUUID(),
 532                      action,
 533                      neutronRouterInterface.getSubnetUUID());
 534         if (!this.enabled) {
 535             return;
 536         }
 537
 538         final boolean isDelete = action == DELETE;
 539
 540         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 541
 542         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 543         // see if they are affected by l3
 544         //
 545         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 546             boolean currPortShouldBeDeleted = false;
 547             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 548             if (isDelete) {
 549                 if (neutronPort.getFixedIPs() != null) {
 550                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 551                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 552                             currPortShouldBeDeleted = true;
 553                             break;
 554                         }
 555                     }
 556                 }
 557             }
 558             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 559         }
 560
 561     }
 562
 563     /**
 564      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 565      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 566      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 567      * way around) on OpenFlow Table 100.
 568      *
 569      * @param actionIn the {@link Action} action to be handled.
 570      * @param neutronFloatingIP An {@link NeutronFloatingIP} instance of NeutronFloatingIP object.
 571      */
 572     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 573                                              Action actionIn) {
 574         Preconditions.checkNotNull(neutronFloatingIP);
 575
 576         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 577                 neutronFloatingIP.getFixedIPAddress(),
 578                 neutronFloatingIP.getFloatingIPAddress(),
 579                 neutronFloatingIP.getFloatingNetworkUUID());
 580         if (!this.enabled) {
 581             return;
 582         }
 583
 584         Action action;
 585
 586         // Consider action to be delete if getFixedIPAddress is null
 587         //
 588         if (neutronFloatingIP.getFixedIPAddress() == null) {
 589             action = DELETE;
 590         } else {
 591             action = actionIn;
 592         }
 593
 594         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 595
 596         if (action != DELETE) {
 597             // must be first, as it updates floatIpDataMapCache
 598             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 599
 600             programFlowsForFloatingIPInbound(neutronFloatingIP, ADD);
 601             programFlowsForFloatingIPOutbound(neutronFloatingIP, ADD);
 602         } else {
 603             programFlowsForFloatingIPOutbound(neutronFloatingIP, DELETE);
 604             programFlowsForFloatingIPInbound(neutronFloatingIP, DELETE);
 605
 606             // must be last, as it updates floatIpDataMapCache
 607             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 608         }
 609     }
 610
 611     /**
 612      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 613      * ip, otherwise for newer one.
 614      */
 615     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 616         Preconditions.checkNotNull(neutronFloatingIP);
 617
 618         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 619         if (fid == null) {
 620             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 621                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 622             return;
 623         }
 624         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 625                                       action);
 626     }
 627
 628     /**
 629      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 630      * ip, otherwise for newer one.
 631      */
 632     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 633         Preconditions.checkNotNull(neutronFloatingIP);
 634
 635         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 636         if (fid == null) {
 637             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 638                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 639             return;
 640         }
 641         programOutboundIpRewriteStage1(fid, action);
 642     }
 643
 644     private void flushExistingIpRewrite() {
 645         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 646             programOutboundIpRewriteStage1(fid, DELETE);
 647         }
 648     }
 649
 650     private void rebuildExistingIpRewrite() {
 651         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 652             programOutboundIpRewriteStage1(fid, ADD);
 653         }
 654     }
 655
 656     /**
 657      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 658      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 659      * requests.
 660      */
 661     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 662         Preconditions.checkNotNull(neutronFloatingIP);
 663         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 664         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 665
 666         // find bridge Node where floating ip is configured by looking up cache for its port
 667         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 668         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 669         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 670         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 671         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 672         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 673
 674         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 675         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 676                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 677         final String providerSegmentationId = tenantNeutronNetwork != null ?
 678                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 679         final String neutronRouterMac = tenantNeutronNetwork != null ?
 680                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 681
 682         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 683                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 684                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 685                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 686             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 687                     fixedIpAddress,
 688                     floatingIpAddress,
 689                     neutronPortForFloatIp,
 690                     neutronTenantPortUuid,
 691                     providerSegmentationId,
 692                     floatingIpMac,
 693                     neutronRouterMac);
 694             return;
 695         }
 696
 697         // get ofport for patch port in br-int
 698         final Long dpId = nodeIfPair.getLeft();
 699         final Long ofPort = findOFPortForExtPatch(dpId);
 700         if (ofPort == null) {
 701             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 702                     dpId);
 703             return;
 704         }
 705
 706         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 707         //
 708         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 709                 ADD)) {
 710             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 711                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 712             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 713             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 714                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 715                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 716         }
 717     }
 718
 719     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 720         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 721         if (floatIpData == null) {
 722             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 723             return;
 724         }
 725
 726         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 727                 floatIpData.floatingIpAddress, DELETE)) {
 728             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 729             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 730                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 731         }
 732     }
 733
 734     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 735         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 736             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 737                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 738                 return neutronPort;
 739             }
 740         }
 741         return null;
 742     }
 743
 744     private Long findOFPortForExtPatch(Long dpId) {
 745         final String brInt = configurationService.getIntegrationBridgeName();
 746         final String brExt = configurationService.getExternalBridgeName();
 747         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 748
 749         Preconditions.checkNotNull(dpId);
 750         Preconditions.checkNotNull(portNameInt);
 751
 752         final long dpidPrimitive = dpId;
 753         for (Node node : nodeCacheManager.getBridgeNodes()) {
 754             if (dpidPrimitive == southbound.getDataPathId(node)) {
 755                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 756                         southbound.getTerminationPointOfBridge(node, portNameInt);
 757                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 758             }
 759         }
 760         return null;
 761     }
 762
 763     /**
 764      * Process the event.
 765      *
 766      * @param action the {@link Action} action to be handled.
 767      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronFloatingIP object.
 768      */
 769     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 770         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 771         if (action == UPDATE) {
 772             this.updateNetworkInCleanupCache(neutronNetwork);
 773         }
 774     }
 775
 776     //
 777     // Callbacks from OVSDB's southbound handler
 778     //
 779     /**
 780      * Process the event.
 781      *
 782      * @param bridgeNode An instance of Node object.
 783      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 784      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 785      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronNetwork
 786      * object.
 787      * @param action the {@link Action} action to be handled.
 788      */
 789     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 790                                      final NeutronNetwork neutronNetwork, Action action) {
 791         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 792                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 793
 794         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 795         if (action != DELETE && neutronPort != null) {
 796             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 797             storePortInCleanupCache(neutronPort);
 798         }
 799
 800         if (!this.enabled) {
 801             return;
 802         }
 803
 804         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 805         final Uuid interfaceUuid = intf.getInterfaceUuid();
 806
 807         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 808                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 809
 810         if (neutronPort != null) {
 811             final String neutronPortUuid = neutronPort.getPortUUID();
 812
 813             if (action != DELETE && dpId != null && interfaceUuid != null) {
 814                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 815             }
 816
 817             handleNeutronPortEvent(neutronPort, action);
 818         }
 819
 820         if (action == DELETE && interfaceUuid != null) {
 821             handleInterfaceEventDelete(intf, dpId);
 822         }
 823     }
 824
 825     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 826         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 827         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 828                 neutronPortUuid, dpId, interfaceUuid.getValue());
 829     }
 830
 831     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 832         // Remove entry from neutronPortToDpIdCache based on interface uuid
 833         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 834             final String currPortUuid = entry.getKey();
 835             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 836                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 837                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 838                 neutronPortToDpIdCache.remove(currPortUuid);
 839                 break;
 840             }
 841         }
 842     }
 843
 844     //
 845     // Internal helpers
 846     //
 847     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 848
 849         final String networkUUID = neutronPort.getNetworkUUID();
 850         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 851
 852         if(!isDelete) {
 853             // If there is no router interface handling the networkUUID, we are done
 854             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 855                 return;
 856             }
 857
 858             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 859             // router interface are handled via handleNeutronRouterInterfaceEvent.
 860             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 861                 return;
 862             }
 863         }
 864
 865         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 866         final String providerSegmentationId = neutronNetwork != null ?
 867                                               neutronNetwork.getProviderSegmentationID() : null;
 868         final String tenantMac = neutronPort.getMacAddress();
 869
 870         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 871             tenantMac == null || tenantMac.isEmpty()) {
 872             // done: go no further w/out all the info needed...
 873             return;
 874         }
 875
 876         final Action action = isDelete ? DELETE : ADD;
 877         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 878         if (nodes.isEmpty()) {
 879             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 880         }
 881         for (Node node : nodes) {
 882             final Long dpid = getDpidForIntegrationBridge(node);
 883             if (dpid == null) {
 884                 continue;
 885             }
 886             if (neutronPort.getFixedIPs() == null) {
 887                 continue;
 888             }
 889             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 890                 final String tenantIpStr = neutronIP.getIpAddress();
 891                 if (tenantIpStr.isEmpty()) {
 892                     continue;
 893                 }
 894
 895                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 896                 // still needed when routing to subnets non-local to node (bug 2076).
 897                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 898             }
 899         }
 900     }
 901
 902     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 903         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 904         /**
 905          * Get updated data and original data for the the changed. Identify the security groups that got
 906          * added and removed and call the appropriate providers for updating the flows.
 907          */
 908         try {
 909             NeutronPort originalPort = neutronPort.getOriginalPort();
 910             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 911                                                                             neutronPort.getOriginalPort());
 912             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 913                                                                               neutronPort);
 914
 915             if (null != addedGroup && !addedGroup.isEmpty()) {
 916                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 917             }
 918             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 919                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 920             }
 921
 922         } catch (Exception e) {
 923             LOG.error("Exception in processSecurityGroupUpdate", e);
 924         }
 925     }
 926
 927     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 928         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 929         if (port1 == null) {
 930             return null;
 931         }
 932         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 933         if (port2 == null) {
 934             return list1;
 935         }
 936         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 937         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 938             NeutronSecurityGroup securityGroup1 = iterator.next();
 939             for (NeutronSecurityGroup securityGroup2 :list2) {
 940                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 941                     iterator.remove();
 942                 }
 943             }
 944         }
 945         return list1;
 946     }
 947
 948     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 949                                            String macAddress, String ipStr,
 950                                            Action actionForNode) {
 951         if (actionForNode == DELETE) {
 952             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 953                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 954         }
 955         if (actionForNode == ADD) {
 956             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 957                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 958         }
 959
 960         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 961                                                        macAddress, ipStr, actionForNode);
 962     }
 963
 964     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 965                                              String macAddress,
 966                                              String address,
 967                                              Action actionForNode) {
 968         Status status;
 969         try {
 970             InetAddress inetAddress = InetAddress.getByName(address);
 971             status = l3ForwardingProvider == null ?
 972                      new Status(StatusCode.SUCCESS) :
 973                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 974                                                                       inetAddress, macAddress, actionForNode);
 975         } catch (UnknownHostException e) {
 976             status = new Status(StatusCode.BADREQUEST);
 977         }
 978
 979         if (status.isSuccess()) {
 980             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 981                          l3ForwardingProvider == null ? "skipped" : "programmed",
 982                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 983         } else {
 984             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 985                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 986         }
 987         return status;
 988     }
 989
 990     // --
 991
 992     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 993                                                        Boolean isDelete) {
 994         Preconditions.checkNotNull(destNeutronRouterInterface);
 995
 996         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 997         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 998         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 999         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
1000         final NeutronNetwork neutronNetwork = subnet != null ?
1001                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
1002         final String destinationSegmentationId = neutronNetwork != null ?
1003                                                  neutronNetwork.getProviderSegmentationID() : null;
1004         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
1005         final String cidr = subnet != null ? subnet.getCidr() : null;
1006         final int mask = getMaskLenFromCidr(cidr);
1007
1008         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1009                      destNeutronRouterInterface, isDelete);
1010
1011         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1012         // the local cache
1013         if (neutronNetwork != null) {
1014             if (macAddress == null || macAddress.isEmpty()) {
1015                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1016             }
1017             if (ipList == null || ipList.isEmpty()) {
1018                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1019             }
1020         }
1021
1022         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1023             cidr == null || cidr.isEmpty() ||
1024             macAddress == null || macAddress.isEmpty() ||
1025             ipList == null || ipList.isEmpty()) {
1026             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1027                          destinationSegmentationId, cidr, macAddress, ipList);
1028             // done: go no further w/out all the info needed...
1029             return;
1030         }
1031
1032         final Action actionForNode = isDelete ? DELETE : ADD;
1033
1034         // Keep cache for finding router's mac from network uuid -- add
1035         //
1036         if (! isDelete) {
1037             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1038             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1039             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1040         }
1041
1042         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1043         if (nodes.isEmpty()) {
1044             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1045         }
1046         for (Node node : nodes) {
1047             final Long dpid = getDpidForIntegrationBridge(node);
1048             if (dpid == null) {
1049                 continue;
1050             }
1051
1052             for (Neutron_IPs neutronIP : ipList) {
1053                 final String ipStr = neutronIP.getIpAddress();
1054                 if (ipStr.isEmpty()) {
1055                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1056                             node.getNodeId().getValue(), ipStr);
1057                     continue;
1058                 }
1059
1060                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1061                 //
1062                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1063                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1064                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1065                                                               neutronNetwork, destinationSegmentationId,
1066                                                               macAddress, ipStr, mask, actionForNode,
1067                                                               true /*isReflexsive*/);
1068                 }
1069
1070                 if (! isExternal) {
1071                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1072                             actionForNode);
1073                 }
1074                 // Enable ARP responder by default, because router interface needs to be responded always.
1075                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1076                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1077             }
1078
1079             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1080             // for the external neutron networks.
1081             //
1082             {
1083                 final Action actionForRewriteExclusion = isExternal ? DELETE : actionForNode;
1084                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1085             }
1086         }
1087
1088         if (isDelete) {
1089             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1090             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1091             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1092         }
1093     }
1094
1095     private void programFlowForNetworkFromExternal(final Node node,
1096                                                    final Long dpid,
1097                                                    final String destinationSegmentationId,
1098                                                    final String dstMacAddress,
1099                                                    final String destIpStr,
1100                                                    final int destMask,
1101                                                    final Action actionForNode) {
1102         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1103                 dstMacAddress, destIpStr, destMask, actionForNode);
1104     }
1105
1106     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1107                                                            final Long dpid,
1108                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1109                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1110                                                            final NeutronNetwork dstNeutronNetwork,
1111                                                            final String destinationSegmentationId,
1112                                                            final String dstMacAddress,
1113                                                            final String destIpStr,
1114                                                            final int destMask,
1115                                                            final Action actionForNode,
1116                                                            Boolean isReflexsive) {
1117         Preconditions.checkNotNull(srcNeutronRouterInterface);
1118         Preconditions.checkNotNull(dstNeutronRouterInterface);
1119
1120         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1121         if (sourceSubnetId == null) {
1122             LOG.error("Could not get provider Subnet ID from router interface {}",
1123                          srcNeutronRouterInterface.getID());
1124             return;
1125         }
1126
1127         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1128         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1129         if (sourceNetworkId == null) {
1130             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1131             return;
1132         }
1133
1134         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1135         if (sourceNetwork == null) {
1136             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1137             return;
1138         }
1139
1140         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1141             // Isolate subnets from different tenants within the same router
1142             return;
1143         }
1144         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1145         if (sourceSegmentationId == null) {
1146             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1147             return;
1148         }
1149         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1150             // Skip 'self'
1151             return;
1152         }
1153
1154         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1155                                      dstMacAddress, destIpStr, destMask, actionForNode);
1156
1157         // Flip roles src->dst; dst->src
1158         if (isReflexsive) {
1159             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1160             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1161             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1162             final String cidr2 = sourceSubnet.getCidr();
1163             final int mask2 = getMaskLenFromCidr(cidr2);
1164
1165             if (cidr2 == null || cidr2.isEmpty() ||
1166                 macAddress2 == null || macAddress2.isEmpty() ||
1167                 ipList2 == null || ipList2.isEmpty()) {
1168                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1169                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1170                 // done: go no further w/out all the info needed...
1171                 return;
1172             }
1173
1174             for (Neutron_IPs neutronIP2 : ipList2) {
1175                 final String ipStr2 = neutronIP2.getIpAddress();
1176                 if (ipStr2.isEmpty()) {
1177                     continue;
1178                 }
1179                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1180                                                           srcNeutronRouterInterface,
1181                                                           sourceNetwork, sourceSegmentationId,
1182                                                           macAddress2, ipStr2, mask2, actionForNode,
1183                                                           false /*isReflexsive*/);
1184             }
1185         }
1186     }
1187
1188     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1189                                               String destinationSegmentationId,
1190                                               String macAddress, String ipStr, int mask,
1191                                               Action actionForNode) {
1192         if (actionForNode == DELETE) {
1193             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1194                          " action {}",
1195                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1196                          macAddress, ipStr, mask, actionForNode);
1197         }
1198         if (actionForNode == ADD) {
1199             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1200                          " action {}",
1201                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1202                          macAddress, ipStr, mask, actionForNode);
1203         }
1204
1205         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1206                                                           macAddress, ipStr, mask, actionForNode);
1207     }
1208
1209     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1210                                                 String destinationSegmentationId,
1211                                                 String macAddress,
1212                                                 String address, int mask,
1213                                                 Action actionForNode) {
1214         Status status;
1215         try {
1216             InetAddress inetAddress = InetAddress.getByName(address);
1217             status = routingProvider == null ?
1218                      new Status(StatusCode.SUCCESS) :
1219                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1220                                                             macAddress, inetAddress, mask, actionForNode);
1221         } catch (UnknownHostException e) {
1222             status = new Status(StatusCode.BADREQUEST);
1223         }
1224
1225         if (status.isSuccess()) {
1226             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1227                          routingProvider == null ? "skipped" : "programmed",
1228                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1229                          actionForNode);
1230         } else {
1231             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1232                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1233                          actionForNode, status);
1234         }
1235         return status;
1236     }
1237
1238     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1239                                            String macAddress, String ipStr,
1240                                            Action action) {
1241         if (action == DELETE ) {
1242             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1243                     dpid, segOrOfPort, macAddress, ipStr, action);
1244         }
1245         if (action == ADD) {
1246             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1247                     dpid, segOrOfPort, macAddress, ipStr, action);
1248         }
1249
1250         Status status = new Status(StatusCode.UNSUPPORTED);
1251         if (icmpEchoProvider != null){
1252             try {
1253                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1254                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1255                                                 macAddress, inetAddress, action);
1256             } catch (UnknownHostException e) {
1257                 status = new Status(StatusCode.BADREQUEST);
1258             }
1259         }
1260
1261         if (status.isSuccess()) {
1262             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1263                     icmpEchoProvider == null ? "skipped" : "programmed",
1264                     macAddress, ipStr, dpid, segOrOfPort, action);
1265         } else {
1266             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1267                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1268         }
1269
1270         return status.isSuccess();
1271     }
1272
1273     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1274                                                   String matchAddress, String rewriteAddress,
1275                                                   Action action) {
1276         if (action == DELETE ) {
1277             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1278                     " action {}",
1279                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1280         }
1281         if (action == ADD ) {
1282             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1283                     " action {}",
1284                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1285         }
1286
1287         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1288                 rewriteAddress, action);
1289         return status.isSuccess();
1290     }
1291
1292     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1293                                                  String matchAddress, String rewriteAddress,
1294                                                  Action action) {
1295         Status status;
1296         try {
1297             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1298             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1299             status = inboundNatProvider == null ?
1300                     new Status(StatusCode.SUCCESS) :
1301                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1302                             inetMatchAddress, inetRewriteAddress,
1303                             action);
1304         } catch (UnknownHostException e) {
1305             status = new Status(StatusCode.BADREQUEST);
1306         }
1307
1308         if (status.isSuccess()) {
1309             final boolean isSkipped = inboundNatProvider == null;
1310             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1311                     isSkipped ? "skipped" : "programmed",
1312                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1313         } else {
1314             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1315                          " status:{}",
1316                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1317                     status);
1318         }
1319         return status;
1320     }
1321
1322     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1323                                                  Action actionForRewriteExclusion) {
1324         if (actionForRewriteExclusion == DELETE ) {
1325             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1326                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1327         }
1328         if (actionForRewriteExclusion == ADD) {
1329             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1330                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1331         }
1332
1333         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1334     }
1335
1336     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1337                                                    Action actionForNode) {
1338         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1339                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1340
1341         if (status.isSuccess()) {
1342             final boolean isSkipped = outboundNatProvider == null;
1343             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1344                          isSkipped ? "skipped" : "programmed",
1345                          cidr, node.getNodeId().getValue(), actionForNode);
1346         } else {
1347             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1348                          cidr, node.getNodeId().getValue(), actionForNode, status);
1349         }
1350         return status;
1351     }
1352
1353     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1354
1355         if (action == DELETE) {
1356             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1357                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1358         }
1359         if (action == ADD) {
1360             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1361                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1362         }
1363
1364         this.programOutboundIpRewriteStage2(fid, action);
1365     }
1366
1367     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1368         Status status;
1369         try {
1370             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1371             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1372             status = outboundNatProvider == null ?
1373                     new Status(StatusCode.SUCCESS) :
1374                     outboundNatProvider.programIpRewriteRule(
1375                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1376                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1377         } catch (UnknownHostException e) {
1378             status = new Status(StatusCode.BADREQUEST);
1379         }
1380
1381         if (status.isSuccess()) {
1382             final boolean isSkipped = outboundNatProvider == null;
1383             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1384                             " action {}",
1385                          isSkipped ? "skipped" : "programmed",
1386                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1387         } else {
1388             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1389                          " action {} status:{}",
1390                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1391         }
1392         return status;
1393     }
1394
1395     private int getMaskLenFromCidr(String cidr) {
1396         if (cidr == null) {
1397             return 0;
1398         }
1399         String[] splits = cidr.split("/");
1400         if (splits.length != 2) {
1401             return 0;
1402         }
1403
1404         int result;
1405         try {
1406             result = Integer.parseInt(splits[1].trim());
1407         } catch (NumberFormatException nfe) {
1408             result = 0;
1409         }
1410         return result;
1411     }
1412
1413     private Long getDpidForIntegrationBridge(Node node) {
1414         // Check if node is integration bridge; and only then return its dpid
1415         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1416             return southbound.getDataPathId(node);
1417         }
1418         return null;
1419     }
1420
1421     private Long getDpidForExternalBridge(Node node) {
1422         // Check if node is external bridge; and only then return its dpid
1423         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1424             return southbound.getDataPathId(node);
1425         }
1426         return null;
1427     }
1428
1429     private Node getExternalBridgeNode(){
1430         //Pickup the first node that has external bridge (br-ex).
1431         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1432         //the external network is reachable from every br-ex
1433         // TODO: Consider other deployment scenario, and thing of better solution.
1434         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1435         for(Node node : allBridges){
1436             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1437                 return node;
1438             }
1439         }
1440         return null;
1441     }
1442
1443     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1444         if (gatewayPort.getFixedIPs() == null) {
1445             return null;
1446         }
1447         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1448             String subnetUUID = neutronIPs.getSubnetUUID();
1449             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1450             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1451                 return extSubnet;
1452             }
1453             if (extSubnet == null) {
1454                 // TODO: when subnet is created, try again.
1455                 LOG.debug("subnet {} in not found", subnetUUID);
1456              }
1457         }
1458         return null;
1459     }
1460
1461     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1462
1463         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1464         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1465             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1466                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1467                     handleNeutronFloatingIPEvent(neutronFloatingIP, DELETE);
1468                 }
1469             }
1470         }
1471     }
1472
1473     private void updateFloatingIPRules(final NeutronPort neutronPort) {
1474         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1475         if (neutronFloatingIps != null) {
1476             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1477                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1478                     handleNeutronFloatingIPEvent(neutronFloatingIP, UPDATE);
1479                 }
1480             }
1481         }
1482     }
1483
1484
1485     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1486
1487         Preconditions.checkNotNull(gatewayPort);
1488         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1489
1490         if(externalNetwork != null){
1491             if(externalNetwork.isRouterExternal()){
1492                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1493
1494                 // TODO: address IPv6 case.
1495                 if (externalSubnet != null &&
1496                         externalSubnet.getIpVersion() == 4 &&
1497                         gatewayPort.getFixedIPs() != null) {
1498                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1499                     Neutron_IPs neutronIP = null;
1500                     for (Neutron_IPs nIP : gatewayPort.getFixedIPs()) {
1501                         InetAddress ipAddress;
1502                         try {
1503                             ipAddress = InetAddress.getByName(nIP.getIpAddress());
1504                         } catch (UnknownHostException e) {
1505                             LOG.warn("unknown host exception {}", e);
1506                             continue;
1507                         }
1508                         if (ipAddress instanceof Inet4Address) {
1509                             neutronIP = nIP;
1510                             break;
1511                         }
1512                     }
1513                     if (neutronIP == null) {
1514                         // TODO IPv6 neighbor discovery
1515                         LOG.debug("Ignoring gateway ports with IPv6 only fixed ip {}",
1516                                   gatewayPort.getFixedIPs());
1517                     } else {
1518                         gatewayMacResolver.resolveMacAddress(
1519                             this, /* gatewayMacResolverListener */
1520                             null, /* externalNetworkBridgeDpid */
1521                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1522                             new Ipv4Address(externalSubnet.getGatewayIP()),
1523                             new Ipv4Address(neutronIP.getIpAddress()),
1524                             new MacAddress(gatewayPort.getMacAddress()),
1525                             true /* periodicRefresh */);
1526                     }
1527                 } else {
1528                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1529                 }
1530             }
1531         }else{
1532             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1533         }
1534     }
1535
1536
1537     private void storePortInCleanupCache(NeutronPort port) {
1538         this.portCleanupCache.put(port.getPortUUID(),port);
1539     }
1540
1541
1542     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1543         removePortFromCleanupCache(originalPort);
1544         storePortInCleanupCache(updatedPort);
1545     }
1546
1547     public void removePortFromCleanupCache(NeutronPort port) {
1548         if(port != null) {
1549             this.portCleanupCache.remove(port.getPortUUID());
1550         }
1551     }
1552
1553     public Map<String, NeutronPort> getPortCleanupCache() {
1554         return this.portCleanupCache;
1555     }
1556
1557     public NeutronPort getPortFromCleanupCache(String portid) {
1558         for (String neutronPortUuid : this.portCleanupCache.keySet()) {
1559             if (neutronPortUuid.equals(portid)) {
1560                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1561                 return this.portCleanupCache.get(neutronPortUuid);
1562             }
1563         }
1564         return null;
1565     }
1566
1567     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1568         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1569     }
1570
1571
1572     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1573         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1574             if (neutronNetworkUuid.equals(network.getNetworkUUID())) {
1575                 this.networkCleanupCache.remove(neutronNetworkUuid);
1576             }
1577         }
1578         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1579     }
1580
1581     public void removeNetworkFromCleanupCache(String networkid) {
1582         NeutronNetwork network = null;
1583         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1584             if (neutronNetworkUuid.equals(networkid)) {
1585                 network = networkCleanupCache.get(neutronNetworkUuid);
1586                 break;
1587             }
1588         }
1589         if (network != null) {
1590             for (String neutronPortUuid:this.portCleanupCache.keySet()) {
1591                 if (this.portCleanupCache.get(neutronPortUuid).getNetworkUUID().equals(network.getNetworkUUID())) {
1592                     LOG.info("This network is used by another port", network);
1593                     return;
1594                 }
1595             }
1596             this.networkCleanupCache.remove(network.getNetworkUUID());
1597         }
1598     }
1599
1600     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1601         return this.networkCleanupCache;
1602     }
1603
1604     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1605         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1606             if (neutronNetworkUuid.equals(networkid)) {
1607                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", networkid);
1608                 return networkCleanupCache.get(neutronNetworkUuid);
1609             }
1610         }
1611         return null;
1612     }
1613     /**
1614      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1615      *
1616      * @param ofPort the OF port number
1617      * @return the string with encoded OF port (example format "OFPort|999")
1618      */
1619     public static String encodeExcplicitOFPort(Long ofPort) {
1620         return "OFPort|" + ofPort.toString();
1621     }
1622     private void initNetworkCleanUpCache() {
1623         if (this.neutronNetworkCache != null) {
1624             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1625                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1626             }
1627         }
1628     }
1629     private void initPortCleanUpCache() {
1630         if (this.neutronPortCache != null) {
1631             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1632                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1633             }
1634         }
1635     }
1636     @Override
1637     public void setDependencies(ServiceReference serviceReference) {
1638         eventDispatcher =
1639                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1640         eventDispatcher.eventHandlerAdded(serviceReference, this);
1641         tenantNetworkManager =
1642                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1643         configurationService =
1644                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1645         arpProvider =
1646                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1647         inboundNatProvider =
1648                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1649         outboundNatProvider =
1650                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1651         routingProvider =
1652                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1653         l3ForwardingProvider =
1654                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1655         distributedArpService =
1656                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1657         nodeCacheManager =
1658                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1659         southbound =
1660                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1661         gatewayMacResolver =
1662                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1663         securityServicesManager =
1664                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1665
1666         initL3AdapterMembers();
1667     }
1668
1669     @Override
1670     public void setDependencies(Object impl) {
1671         if (impl instanceof INeutronNetworkCRUD) {
1672             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1673             initNetworkCleanUpCache();
1674         } else if (impl instanceof INeutronPortCRUD) {
1675             neutronPortCache = (INeutronPortCRUD)impl;
1676             initPortCleanUpCache();
1677         } else if (impl instanceof INeutronSubnetCRUD) {
1678             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1679         } else if (impl instanceof INeutronFloatingIPCRUD) {
1680             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1681         } else if (impl instanceof ArpProvider) {
1682             arpProvider = (ArpProvider)impl;
1683         } else if (impl instanceof InboundNatProvider) {
1684             inboundNatProvider = (InboundNatProvider)impl;
1685         } else if (impl instanceof OutboundNatProvider) {
1686             outboundNatProvider = (OutboundNatProvider)impl;
1687         } else if (impl instanceof RoutingProvider) {
1688             routingProvider = (RoutingProvider)impl;
1689         } else if (impl instanceof L3ForwardingProvider) {
1690             l3ForwardingProvider = (L3ForwardingProvider)impl;
1691         }else if (impl instanceof GatewayMacResolver) {
1692             gatewayMacResolver = (GatewayMacResolver)impl;
1693         }else if (impl instanceof IcmpEchoProvider) {
1694             icmpEchoProvider = (IcmpEchoProvider)impl;
1695         }
1696
1697         populateL3ForwardingCaches();
1698     }
1699 }