openstack/net-virt/src/main/java/org/opendaylight/netvirt/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.openstack.netvirt.impl;
  10
  11 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.ADD;
  12 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.DELETE;
  13 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.UPDATE;
  14
  15 import com.google.common.base.Preconditions;
  16 import org.apache.commons.lang3.tuple.ImmutablePair;
  17 import org.apache.commons.lang3.tuple.Pair;
  18 import org.opendaylight.netvirt.openstack.netvirt.AbstractEvent;
  19 import org.opendaylight.netvirt.openstack.netvirt.AbstractHandler;
  20 import org.opendaylight.netvirt.openstack.netvirt.ConfigInterface;
  21 import org.opendaylight.netvirt.openstack.netvirt.NeutronL3AdapterEvent;
  22 import org.opendaylight.netvirt.openstack.netvirt.api.Action;
  23 import org.opendaylight.netvirt.openstack.netvirt.api.ArpProvider;
  24 import org.opendaylight.netvirt.openstack.netvirt.api.ConfigurationService;
  25 import org.opendaylight.netvirt.openstack.netvirt.api.Constants;
  26 import org.opendaylight.netvirt.openstack.netvirt.api.EventDispatcher;
  27 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolver;
  28 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolverListener;
  29 import org.opendaylight.netvirt.openstack.netvirt.api.IcmpEchoProvider;
  30 import org.opendaylight.netvirt.openstack.netvirt.api.InboundNatProvider;
  31 import org.opendaylight.netvirt.openstack.netvirt.api.L3ForwardingProvider;
  32 import org.opendaylight.netvirt.openstack.netvirt.api.NodeCacheManager;
  33 import org.opendaylight.netvirt.openstack.netvirt.api.OutboundNatProvider;
  34 import org.opendaylight.netvirt.openstack.netvirt.api.RoutingProvider;
  35 import org.opendaylight.netvirt.openstack.netvirt.api.SecurityServicesManager;
  36 import org.opendaylight.netvirt.openstack.netvirt.api.Southbound;
  37 import org.opendaylight.netvirt.openstack.netvirt.api.Status;
  38 import org.opendaylight.netvirt.openstack.netvirt.api.StatusCode;
  39 import org.opendaylight.netvirt.openstack.netvirt.api.TenantNetworkManager;
  40 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronFloatingIP;
  41 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronNetwork;
  42 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter;
  43 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter_Interface;
  44 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSecurityGroup;
  45 import org.opendaylight.netvirt.openstack.netvirt.translator.Neutron_IPs;
  46 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  47 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  48 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronPortCRUD;
  49 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronPort;
  50 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSubnet;
  51 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  52 import org.opendaylight.netvirt.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  53 import org.opendaylight.netvirt.utils.neutron.utils.NeutronModelsDataStoreHelper;
  54 import org.opendaylight.netvirt.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  64 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  65 import org.osgi.framework.ServiceReference;
  66 import org.slf4j.Logger;
  67 import org.slf4j.LoggerFactory;
  68
  69 import java.net.Inet4Address;
  70 import java.net.InetAddress;
  71 import java.net.UnknownHostException;
  72 import java.util.ArrayList;
  73 import java.util.Collection;
  74 import java.util.HashMap;
  75 import java.util.Iterator;
  76 import java.util.List;
  77 import java.util.Map;
  78
  79 /**
  80  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  81  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  82  * as well as the multi-tenant router forwarding provider.
  83  */
  84 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  85     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  86
  87     // The implementation for each of these services is resolved by the OSGi Service Manager
  88     private volatile ConfigurationService configurationService;
  89     private volatile TenantNetworkManager tenantNetworkManager;
  90     private volatile NodeCacheManager nodeCacheManager;
  91     private volatile INeutronNetworkCRUD neutronNetworkCache;
  92     private volatile INeutronSubnetCRUD neutronSubnetCache;
  93     private volatile INeutronPortCRUD neutronPortCache;
  94     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  95     private volatile L3ForwardingProvider l3ForwardingProvider;
  96     private volatile InboundNatProvider inboundNatProvider;
  97     private volatile OutboundNatProvider outboundNatProvider;
  98     private volatile ArpProvider arpProvider;
  99     private volatile RoutingProvider routingProvider;
 100     private volatile GatewayMacResolver gatewayMacResolver;
 101     private volatile SecurityServicesManager securityServicesManager;
 102     private volatile IcmpEchoProvider icmpEchoProvider;
 103
 104     private class FloatIpData {
 105         // br-int of node where floating ip is associated with tenant port
 106         private final Long dpid;
 107         // patch port in br-int used to reach br-ex
 108         private final Long ofPort;
 109         // segmentation id of the net where fixed ip is instantiated
 110         private final String segId;
 111         // mac address assigned to neutron port of floating ip
 112         private final String macAddress;
 113         private final String floatingIpAddress;
 114         // ip address given to tenant vm
 115         private final String fixedIpAddress;
 116         private final String neutronRouterMac;
 117
 118         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 119                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 120             this.dpid = dpid;
 121             this.ofPort = ofPort;
 122             this.segId = segId;
 123             this.macAddress = macAddress;
 124             this.floatingIpAddress = floatingIpAddress;
 125             this.fixedIpAddress = fixedIpAddress;
 126             this.neutronRouterMac = neutronRouterMac;
 127         }
 128     }
 129
 130     private Map<String, String> networkIdToRouterMacCache;
 131     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 132     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 133
 134     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 135     private Map<String, FloatIpData> floatIpDataMapCache;
 136
 137     private String externalRouterMac;
 138     private Boolean enabled = false;
 139     private Boolean isCachePopulationDone = false;
 140     private Map<String, NeutronPort> portCleanupCache;
 141     private Map<String, NeutronNetwork> networkCleanupCache;
 142
 143     private Southbound southbound;
 144     private DistributedArpService distributedArpService;
 145     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 146
 147     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 148     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 149     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 150     private static final String OWNER_FLOATING_IP = "network:floatingip";
 151     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 152
 153     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 154         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 155         this.neutronModelsDataStoreHelper = neutronHelper;
 156     }
 157
 158     private void initL3AdapterMembers() {
 159         Preconditions.checkNotNull(configurationService);
 160
 161         if (configurationService.isL3ForwardingEnabled()) {
 162             this.networkIdToRouterMacCache = new HashMap<>();
 163             this.networkIdToRouterIpListCache = new HashMap<>();
 164             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 165             this.neutronPortToDpIdCache = new HashMap<>();
 166             this.floatIpDataMapCache = new HashMap<>();
 167
 168             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 169             if (this.externalRouterMac == null) {
 170                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 171             }
 172             this.enabled = true;
 173             LOG.info("OVSDB L3 forwarding is enabled");
 174         } else {
 175             LOG.debug("OVSDB L3 forwarding is disabled");
 176         }
 177         this.portCleanupCache = new HashMap<>();
 178         this.networkCleanupCache = new HashMap<>();
 179     }
 180
 181     //
 182     // Callbacks from AbstractHandler
 183     //
 184     @Override
 185     public void processEvent(AbstractEvent abstractEvent) {
 186         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 187             LOG.error("Unable to process abstract event " + abstractEvent);
 188             return;
 189         }
 190         if (!this.enabled) {
 191             return;
 192         }
 193
 194         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 195         switch (ev.getAction()) {
 196             case UPDATE:
 197                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 198                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 199                 } else {
 200                     LOG.warn("Received update for an unexpected event " + ev);
 201                 }
 202                 break;
 203             case ADD:
 204                 // fall through...
 205                 // break;
 206             case DELETE:
 207                 // fall through...
 208                 // break;
 209             default:
 210                 LOG.warn("Unable to process event " + ev);
 211                 break;
 212         }
 213     }
 214
 215     //
 216     // Callbacks from GatewayMacResolverListener
 217     //
 218
 219     @Override
 220     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 221         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 222                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 223         if (!this.enabled) {
 224             return;
 225         }
 226
 227         if (macAddress == null || macAddress.getValue() == null) {
 228             // TODO: handle cases when mac is null
 229             return;
 230         }
 231
 232         //
 233         // Enqueue event so update is handled by adapter's thread
 234         //
 235         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 236     }
 237
 238     private void populateL3ForwardingCaches() {
 239         if (!this.enabled) {
 240             return;
 241         }
 242         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 243                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 244             return;
 245         }
 246         this.isCachePopulationDone = true;
 247         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 248         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 249         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 250         if(routers != null && routers.getRouter() != null && ports != null) {
 251             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 252             for( Router router : routers.getRouter()) {
 253                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 254                 if(!ports.getPort().isEmpty()) {
 255                     for( Port port : ports.getPort()) {
 256                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 257                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 258                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 259                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 260                                     , port.getMacAddress());
 261
 262                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 263                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 264                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 265                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 266                         }
 267                     }
 268                 }else {
 269                     LOG.warn("L3 Cache Population :Did not find any port information " +
 270                             "in config Data Store for router {}",router);
 271                 }
 272             }
 273         }
 274         LOG.debug("NetVirt L3 caches population is done");
 275     }
 276
 277     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 278         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 279             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 280             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 281             for(Node bridge : bridges) {
 282                 List<OvsdbTerminationPointAugmentation> interfaces
 283                         = southbound.extractTerminationPointAugmentations(bridge);
 284                 if(interfaces != null && !interfaces.isEmpty()) {
 285                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 286                             interfaces.size(), bridge.getNodeId());
 287                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 288                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 289                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 290                             Long dpId = getDpidForIntegrationBridge(bridge);
 291                             Uuid interfaceUuid = intf.getInterfaceUuid();
 292                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 293                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 294                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 295                             break;
 296                         }
 297                     }
 298                 }
 299             }
 300         }
 301         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 302     }
 303
 304     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 305         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 306         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 307         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 308             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 309                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 310                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 311                             "Fetching from data store.",neutronFloatingIP.getID());
 312                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 313                 }
 314             }
 315         }
 316         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 317         return floatIpDataMapCache.values();
 318     }
 319     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 320         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 321                 "floating id {} ",neutronFloatingId);
 322         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 323             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 324             if (neutronFloatingIP == null) {
 325                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 326                 return null;
 327             }
 328             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 329             NeutronPort neutronPortForFloatIp = null;
 330             for (NeutronPort neutronPort : neutronPorts) {
 331                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 332                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 333                     neutronPortForFloatIp = neutronPort;
 334                     break;
 335                 }
 336             }
 337
 338             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 339             if(neutronTenantPortUuid == null) {
 340                 return null;
 341             }
 342             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 343             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 344             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 345             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 346
 347             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 348             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 349                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 350             String providerSegmentationId = tenantNeutronNetwork != null ?
 351                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 352             String neutronRouterMac = tenantNeutronNetwork != null ?
 353                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 354
 355             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 356                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 357                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 358                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 359                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 360                                 "seg {} mac {} rtrMac {}",
 361                         fixedIpAddress,
 362                         floatingIpAddress,
 363                         neutronPortForFloatIp,
 364                         neutronTenantPortUuid,
 365                         providerSegmentationId,
 366                         floatingIpMac,
 367                         neutronRouterMac);
 368
 369                 return null;
 370             }
 371
 372             // get ofport for patch port in br-int
 373             final Long dpId = nodeIfPair.getLeft();
 374             final Long ofPort = findOFPortForExtPatch(dpId);
 375             if (ofPort == null) {
 376                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 377                                 "to connect floating ip to external bridge. dpid {}",
 378                         dpId);
 379                 return null;
 380             }
 381
 382             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 383                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 384             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 385
 386         }
 387         return floatIpDataMapCache.get(neutronFloatingId);
 388     }
 389     /**
 390      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 391      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 392      * subscribe so that the mac address associated to the gateway ip address is available.
 393      *
 394      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 395      */
 396     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 397         Preconditions.checkNotNull(externalRouterMacUpdate);
 398
 399         flushExistingIpRewrite();
 400         this.externalRouterMac = externalRouterMacUpdate;
 401         rebuildExistingIpRewrite();
 402     }
 403
 404     /**
 405      * Process the event.
 406      *
 407      * @param action the {@link Action} action to be handled.
 408      * @param subnet An instance of NeutronSubnet object.
 409      */
 410     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 411         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 412         if (action == ADD) {
 413             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 414         }
 415     }
 416
 417     /**
 418      * Process the port event as a router interface event.
 419      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 420      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 421      *
 422      * @param action the {@link Action} action to be handled.
 423      * @param neutronPort An instance of NeutronPort object.
 424      */
 425     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 426         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 427
 428         if (action == UPDATE) {
 429             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 430             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 431             this.processSecurityGroupUpdate(neutronPort);
 432         }
 433
 434         if (!this.enabled) {
 435             return;
 436         }
 437
 438         final boolean isDelete = action == DELETE;
 439
 440         if (action == DELETE) {
 441             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 442             this.cleanupFloatingIPRules(neutronPort);
 443         }
 444         else if (action == UPDATE){
 445             // Bug 5353: VM restart cause floatingIp flows to be removed
 446             this.updateFloatingIPRules(neutronPort);
 447         }
 448
 449         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 450             if (!isDelete) {
 451                 LOG.info("Port {} is network router gateway interface, "
 452                         + "triggering gateway resolution for the attached external network", neutronPort);
 453                 this.triggerGatewayMacResolver(neutronPort);
 454             }else{
 455                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 456                 if (null == externalNetwork) {
 457                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 458                 }
 459
 460                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 461                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 462                     // TODO support IPv6
 463                     if (externalSubnet != null &&
 464                             externalSubnet.getIpVersion() == 4) {
 465                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 466                     }
 467                 }
 468             }
 469         }
 470
 471         // Treat the port event as a router interface event if the port belongs to router. This is a
 472         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 473         //
 474         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 475             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 476
 477             if (neutronPort.getFixedIPs() != null) {
 478                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 479                     NeutronRouter_Interface neutronRouterInterface =
 480                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 481                     // id of router interface to be same as subnet
 482                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 483                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 484
 485                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 486                 }
 487             }
 488         } else {
 489             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 490             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 491             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 492             // there.
 493             //
 494             if (!isDelete && neutronPort.getFixedIPs() != null) {
 495                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 496                     NeutronRouter_Interface neutronRouterInterface =
 497                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 498                     if (neutronRouterInterface != null) {
 499                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 500                     }
 501                 }
 502             }
 503             this.updateL3ForNeutronPort(neutronPort, isDelete);
 504         }
 505     }
 506
 507     /**
 508      * Process the event.
 509      *
 510      * @param action the {@link Action} action to be handled.
 511      * @param neutronRouter An instance of NeutronRouter object.
 512      */
 513     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 514         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 515     }
 516
 517     /**
 518      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 519      * delete the ports on the same subnet.
 520      *
 521      * @param action the {@link Action} action to be handled.
 522      * @param neutronRouter An instance of NeutronRouter object.
 523      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 524      */
 525     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 526                                                   final NeutronRouter_Interface neutronRouterInterface,
 527                                                   Action action) {
 528         LOG.debug("Router interface {} got event {}. Subnet {}",
 529                      neutronRouterInterface.getPortUUID(),
 530                      action,
 531                      neutronRouterInterface.getSubnetUUID());
 532         if (!this.enabled) {
 533             return;
 534         }
 535
 536         final boolean isDelete = action == DELETE;
 537
 538         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 539
 540         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 541         // see if they are affected by l3
 542         //
 543         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 544             boolean currPortShouldBeDeleted = false;
 545             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 546             if (isDelete) {
 547                 if (neutronPort.getFixedIPs() != null) {
 548                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 549                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 550                             currPortShouldBeDeleted = true;
 551                             break;
 552                         }
 553                     }
 554                 }
 555             }
 556             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 557         }
 558
 559     }
 560
 561     /**
 562      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 563      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 564      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 565      * way around) on OpenFlow Table 100.
 566      *
 567      * @param actionIn the {@link Action} action to be handled.
 568      * @param neutronFloatingIP An {@link NeutronFloatingIP} instance of NeutronFloatingIP object.
 569      */
 570     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 571                                              Action actionIn) {
 572         Preconditions.checkNotNull(neutronFloatingIP);
 573
 574         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 575                 neutronFloatingIP.getFixedIPAddress(),
 576                 neutronFloatingIP.getFloatingIPAddress(),
 577                 neutronFloatingIP.getFloatingNetworkUUID());
 578         if (!this.enabled) {
 579             return;
 580         }
 581
 582         Action action;
 583
 584         // Consider action to be delete if getFixedIPAddress is null
 585         //
 586         if (neutronFloatingIP.getFixedIPAddress() == null) {
 587             action = DELETE;
 588         } else {
 589             action = actionIn;
 590         }
 591
 592         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 593
 594         if (action != DELETE) {
 595             // must be first, as it updates floatIpDataMapCache
 596             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 597
 598             programFlowsForFloatingIPInbound(neutronFloatingIP, ADD);
 599             programFlowsForFloatingIPOutbound(neutronFloatingIP, ADD);
 600         } else {
 601             programFlowsForFloatingIPOutbound(neutronFloatingIP, DELETE);
 602             programFlowsForFloatingIPInbound(neutronFloatingIP, DELETE);
 603
 604             // must be last, as it updates floatIpDataMapCache
 605             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 606         }
 607     }
 608
 609     /**
 610      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 611      * ip, otherwise for newer one.
 612      */
 613     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 614         Preconditions.checkNotNull(neutronFloatingIP);
 615
 616         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 617         if (fid == null) {
 618             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 619                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 620             return;
 621         }
 622         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 623                                       action);
 624     }
 625
 626     /**
 627      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 628      * ip, otherwise for newer one.
 629      */
 630     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 631         Preconditions.checkNotNull(neutronFloatingIP);
 632
 633         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 634         if (fid == null) {
 635             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 636                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 637             return;
 638         }
 639         programOutboundIpRewriteStage1(fid, action);
 640     }
 641
 642     private void flushExistingIpRewrite() {
 643         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 644             programOutboundIpRewriteStage1(fid, DELETE);
 645         }
 646     }
 647
 648     private void rebuildExistingIpRewrite() {
 649         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 650             programOutboundIpRewriteStage1(fid, ADD);
 651         }
 652     }
 653
 654     /**
 655      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 656      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 657      * requests.
 658      */
 659     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 660         Preconditions.checkNotNull(neutronFloatingIP);
 661         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 662         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 663
 664         // find bridge Node where floating ip is configured by looking up cache for its port
 665         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 666         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 667         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 668         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 669         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 670         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 671
 672         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 673         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 674                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 675         final String providerSegmentationId = tenantNeutronNetwork != null ?
 676                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 677         final String neutronRouterMac = tenantNeutronNetwork != null ?
 678                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 679
 680         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 681                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 682                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 683                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 684             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 685                     fixedIpAddress,
 686                     floatingIpAddress,
 687                     neutronPortForFloatIp,
 688                     neutronTenantPortUuid,
 689                     providerSegmentationId,
 690                     floatingIpMac,
 691                     neutronRouterMac);
 692             return;
 693         }
 694
 695         // get ofport for patch port in br-int
 696         final Long dpId = nodeIfPair.getLeft();
 697         final Long ofPort = findOFPortForExtPatch(dpId);
 698         if (ofPort == null) {
 699             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 700                     dpId);
 701             return;
 702         }
 703
 704         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 705         //
 706         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 707                 ADD)) {
 708             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 709                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 710             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 711             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 712                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 713                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 714         }
 715     }
 716
 717     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 718         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 719         if (floatIpData == null) {
 720             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 721             return;
 722         }
 723
 724         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 725                 floatIpData.floatingIpAddress, DELETE)) {
 726             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 727             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 728                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 729         }
 730     }
 731
 732     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 733         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 734             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 735                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 736                 return neutronPort;
 737             }
 738         }
 739         return null;
 740     }
 741
 742     private Long findOFPortForExtPatch(Long dpId) {
 743         final String brInt = configurationService.getIntegrationBridgeName();
 744         final String brExt = configurationService.getExternalBridgeName();
 745         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 746
 747         Preconditions.checkNotNull(dpId);
 748         Preconditions.checkNotNull(portNameInt);
 749
 750         final long dpidPrimitive = dpId;
 751         for (Node node : nodeCacheManager.getBridgeNodes()) {
 752             if (dpidPrimitive == southbound.getDataPathId(node)) {
 753                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 754                         southbound.getTerminationPointOfBridge(node, portNameInt);
 755                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 756             }
 757         }
 758         return null;
 759     }
 760
 761     /**
 762      * Process the event.
 763      *
 764      * @param action the {@link Action} action to be handled.
 765      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronFloatingIP object.
 766      */
 767     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 768         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 769         if (action == UPDATE) {
 770             this.updateNetworkInCleanupCache(neutronNetwork);
 771         }
 772     }
 773
 774     //
 775     // Callbacks from OVSDB's southbound handler
 776     //
 777     /**
 778      * Process the event.
 779      *
 780      * @param bridgeNode An instance of Node object.
 781      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 782      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 783      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronNetwork
 784      * object.
 785      * @param action the {@link Action} action to be handled.
 786      */
 787     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 788                                      final NeutronNetwork neutronNetwork, Action action) {
 789         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 790                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 791
 792         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 793         if (action != DELETE && neutronPort != null) {
 794             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 795             storePortInCleanupCache(neutronPort);
 796         }
 797
 798         if (!this.enabled) {
 799             return;
 800         }
 801
 802         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 803         final Uuid interfaceUuid = intf.getInterfaceUuid();
 804
 805         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 806                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 807
 808         if (neutronPort != null) {
 809             final String neutronPortUuid = neutronPort.getPortUUID();
 810
 811             if (action != DELETE && dpId != null && interfaceUuid != null) {
 812                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 813             }
 814
 815             handleNeutronPortEvent(neutronPort, action);
 816         }
 817
 818         if (action == DELETE && interfaceUuid != null) {
 819             handleInterfaceEventDelete(intf, dpId);
 820         }
 821     }
 822
 823     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 824         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 825         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 826                 neutronPortUuid, dpId, interfaceUuid.getValue());
 827     }
 828
 829     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 830         // Remove entry from neutronPortToDpIdCache based on interface uuid
 831         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 832             final String currPortUuid = entry.getKey();
 833             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 834                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 835                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 836                 neutronPortToDpIdCache.remove(currPortUuid);
 837                 break;
 838             }
 839         }
 840     }
 841
 842     //
 843     // Internal helpers
 844     //
 845     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 846
 847         final String networkUUID = neutronPort.getNetworkUUID();
 848         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 849
 850         if(!isDelete) {
 851             // If there is no router interface handling the networkUUID, we are done
 852             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 853                 return;
 854             }
 855
 856             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 857             // router interface are handled via handleNeutronRouterInterfaceEvent.
 858             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 859                 return;
 860             }
 861         }
 862
 863         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 864         final String providerSegmentationId = neutronNetwork != null ?
 865                                               neutronNetwork.getProviderSegmentationID() : null;
 866         final String tenantMac = neutronPort.getMacAddress();
 867
 868         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 869             tenantMac == null || tenantMac.isEmpty()) {
 870             // done: go no further w/out all the info needed...
 871             return;
 872         }
 873
 874         final Action action = isDelete ? DELETE : ADD;
 875         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 876         if (nodes.isEmpty()) {
 877             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 878         }
 879         for (Node node : nodes) {
 880             final Long dpid = getDpidForIntegrationBridge(node);
 881             if (dpid == null) {
 882                 continue;
 883             }
 884             if (neutronPort.getFixedIPs() == null) {
 885                 continue;
 886             }
 887             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 888                 final String tenantIpStr = neutronIP.getIpAddress();
 889                 if (tenantIpStr.isEmpty()) {
 890                     continue;
 891                 }
 892
 893                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 894                 // still needed when routing to subnets non-local to node (bug 2076).
 895                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 896             }
 897         }
 898     }
 899
 900     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 901         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 902         /**
 903          * Get updated data and original data for the the changed. Identify the security groups that got
 904          * added and removed and call the appropriate providers for updating the flows.
 905          */
 906         try {
 907             NeutronPort originalPort = neutronPort.getOriginalPort();
 908             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 909                                                                             neutronPort.getOriginalPort());
 910             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 911                                                                               neutronPort);
 912
 913             if (null != addedGroup && !addedGroup.isEmpty()) {
 914                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 915             }
 916             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 917                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 918             }
 919
 920         } catch (Exception e) {
 921             LOG.error("Exception in processSecurityGroupUpdate", e);
 922         }
 923     }
 924
 925     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 926         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 927         if (port1 == null) {
 928             return null;
 929         }
 930         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 931         if (port2 == null) {
 932             return list1;
 933         }
 934         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 935         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 936             NeutronSecurityGroup securityGroup1 = iterator.next();
 937             for (NeutronSecurityGroup securityGroup2 :list2) {
 938                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 939                     iterator.remove();
 940                 }
 941             }
 942         }
 943         return list1;
 944     }
 945
 946     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 947                                            String macAddress, String ipStr,
 948                                            Action actionForNode) {
 949         if (actionForNode == DELETE) {
 950             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 951                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 952         }
 953         if (actionForNode == ADD) {
 954             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 955                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 956         }
 957
 958         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 959                                                        macAddress, ipStr, actionForNode);
 960     }
 961
 962     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 963                                              String macAddress,
 964                                              String address,
 965                                              Action actionForNode) {
 966         Status status;
 967         try {
 968             InetAddress inetAddress = InetAddress.getByName(address);
 969             status = l3ForwardingProvider == null ?
 970                      new Status(StatusCode.SUCCESS) :
 971                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 972                                                                       inetAddress, macAddress, actionForNode);
 973         } catch (UnknownHostException e) {
 974             status = new Status(StatusCode.BADREQUEST);
 975         }
 976
 977         if (status.isSuccess()) {
 978             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 979                          l3ForwardingProvider == null ? "skipped" : "programmed",
 980                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 981         } else {
 982             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 983                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 984         }
 985         return status;
 986     }
 987
 988     // --
 989
 990     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 991                                                        Boolean isDelete) {
 992         Preconditions.checkNotNull(destNeutronRouterInterface);
 993
 994         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 995         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 996         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 997         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 998         final NeutronNetwork neutronNetwork = subnet != null ?
 999                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
1000         final String destinationSegmentationId = neutronNetwork != null ?
1001                                                  neutronNetwork.getProviderSegmentationID() : null;
1002         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
1003         final String cidr = subnet != null ? subnet.getCidr() : null;
1004         final int mask = getMaskLenFromCidr(cidr);
1005
1006         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1007                      destNeutronRouterInterface, isDelete);
1008
1009         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1010         // the local cache
1011         if (neutronNetwork != null) {
1012             if (macAddress == null || macAddress.isEmpty()) {
1013                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1014             }
1015             if (ipList == null || ipList.isEmpty()) {
1016                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1017             }
1018         }
1019
1020         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1021             cidr == null || cidr.isEmpty() ||
1022             macAddress == null || macAddress.isEmpty() ||
1023             ipList == null || ipList.isEmpty()) {
1024             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1025                          destinationSegmentationId, cidr, macAddress, ipList);
1026             // done: go no further w/out all the info needed...
1027             return;
1028         }
1029
1030         final Action actionForNode = isDelete ? DELETE : ADD;
1031
1032         // Keep cache for finding router's mac from network uuid -- add
1033         //
1034         if (! isDelete) {
1035             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1036             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1037             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1038         }
1039
1040         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1041         if (nodes.isEmpty()) {
1042             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1043         }
1044         for (Node node : nodes) {
1045             final Long dpid = getDpidForIntegrationBridge(node);
1046             if (dpid == null) {
1047                 continue;
1048             }
1049
1050             for (Neutron_IPs neutronIP : ipList) {
1051                 final String ipStr = neutronIP.getIpAddress();
1052                 if (ipStr.isEmpty()) {
1053                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1054                             node.getNodeId().getValue(), ipStr);
1055                     continue;
1056                 }
1057
1058                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1059                 //
1060                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1061                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1062                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1063                                                               neutronNetwork, destinationSegmentationId,
1064                                                               macAddress, ipStr, mask, actionForNode,
1065                                                               true /*isReflexsive*/);
1066                 }
1067
1068                 if (! isExternal) {
1069                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1070                             actionForNode);
1071                 }
1072                 // Enable ARP responder by default, because router interface needs to be responded always.
1073                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1074                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1075             }
1076
1077             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1078             // for the external neutron networks.
1079             //
1080             {
1081                 final Action actionForRewriteExclusion = isExternal ? DELETE : actionForNode;
1082                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1083             }
1084         }
1085
1086         if (isDelete) {
1087             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1088             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1089             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1090         }
1091     }
1092
1093     private void programFlowForNetworkFromExternal(final Node node,
1094                                                    final Long dpid,
1095                                                    final String destinationSegmentationId,
1096                                                    final String dstMacAddress,
1097                                                    final String destIpStr,
1098                                                    final int destMask,
1099                                                    final Action actionForNode) {
1100         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1101                 dstMacAddress, destIpStr, destMask, actionForNode);
1102     }
1103
1104     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1105                                                            final Long dpid,
1106                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1107                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1108                                                            final NeutronNetwork dstNeutronNetwork,
1109                                                            final String destinationSegmentationId,
1110                                                            final String dstMacAddress,
1111                                                            final String destIpStr,
1112                                                            final int destMask,
1113                                                            final Action actionForNode,
1114                                                            Boolean isReflexsive) {
1115         Preconditions.checkNotNull(srcNeutronRouterInterface);
1116         Preconditions.checkNotNull(dstNeutronRouterInterface);
1117
1118         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1119         if (sourceSubnetId == null) {
1120             LOG.error("Could not get provider Subnet ID from router interface {}",
1121                          srcNeutronRouterInterface.getID());
1122             return;
1123         }
1124
1125         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1126         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1127         if (sourceNetworkId == null) {
1128             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1129             return;
1130         }
1131
1132         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1133         if (sourceNetwork == null) {
1134             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1135             return;
1136         }
1137
1138         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1139             // Isolate subnets from different tenants within the same router
1140             return;
1141         }
1142         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1143         if (sourceSegmentationId == null) {
1144             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1145             return;
1146         }
1147         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1148             // Skip 'self'
1149             return;
1150         }
1151
1152         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1153                                      dstMacAddress, destIpStr, destMask, actionForNode);
1154
1155         // Flip roles src->dst; dst->src
1156         if (isReflexsive) {
1157             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1158             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1159             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1160             final String cidr2 = sourceSubnet.getCidr();
1161             final int mask2 = getMaskLenFromCidr(cidr2);
1162
1163             if (cidr2 == null || cidr2.isEmpty() ||
1164                 macAddress2 == null || macAddress2.isEmpty() ||
1165                 ipList2 == null || ipList2.isEmpty()) {
1166                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1167                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1168                 // done: go no further w/out all the info needed...
1169                 return;
1170             }
1171
1172             for (Neutron_IPs neutronIP2 : ipList2) {
1173                 final String ipStr2 = neutronIP2.getIpAddress();
1174                 if (ipStr2.isEmpty()) {
1175                     continue;
1176                 }
1177                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1178                                                           srcNeutronRouterInterface,
1179                                                           sourceNetwork, sourceSegmentationId,
1180                                                           macAddress2, ipStr2, mask2, actionForNode,
1181                                                           false /*isReflexsive*/);
1182             }
1183         }
1184     }
1185
1186     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1187                                               String destinationSegmentationId,
1188                                               String macAddress, String ipStr, int mask,
1189                                               Action actionForNode) {
1190         if (actionForNode == DELETE) {
1191             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1192                          " action {}",
1193                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1194                          macAddress, ipStr, mask, actionForNode);
1195         }
1196         if (actionForNode == ADD) {
1197             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1198                          " action {}",
1199                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1200                          macAddress, ipStr, mask, actionForNode);
1201         }
1202
1203         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1204                                                           macAddress, ipStr, mask, actionForNode);
1205     }
1206
1207     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1208                                                 String destinationSegmentationId,
1209                                                 String macAddress,
1210                                                 String address, int mask,
1211                                                 Action actionForNode) {
1212         Status status;
1213         try {
1214             InetAddress inetAddress = InetAddress.getByName(address);
1215             status = routingProvider == null ?
1216                      new Status(StatusCode.SUCCESS) :
1217                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1218                                                             macAddress, inetAddress, mask, actionForNode);
1219         } catch (UnknownHostException e) {
1220             status = new Status(StatusCode.BADREQUEST);
1221         }
1222
1223         if (status.isSuccess()) {
1224             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1225                          routingProvider == null ? "skipped" : "programmed",
1226                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1227                          actionForNode);
1228         } else {
1229             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1230                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1231                          actionForNode, status);
1232         }
1233         return status;
1234     }
1235
1236     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1237                                            String macAddress, String ipStr,
1238                                            Action action) {
1239         if (action == DELETE ) {
1240             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1241                     dpid, segOrOfPort, macAddress, ipStr, action);
1242         }
1243         if (action == ADD) {
1244             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1245                     dpid, segOrOfPort, macAddress, ipStr, action);
1246         }
1247
1248         Status status = new Status(StatusCode.UNSUPPORTED);
1249         if (icmpEchoProvider != null){
1250             try {
1251                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1252                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1253                                                 macAddress, inetAddress, action);
1254             } catch (UnknownHostException e) {
1255                 status = new Status(StatusCode.BADREQUEST);
1256             }
1257         }
1258
1259         if (status.isSuccess()) {
1260             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1261                     icmpEchoProvider == null ? "skipped" : "programmed",
1262                     macAddress, ipStr, dpid, segOrOfPort, action);
1263         } else {
1264             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1265                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1266         }
1267
1268         return status.isSuccess();
1269     }
1270
1271     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1272                                                   String matchAddress, String rewriteAddress,
1273                                                   Action action) {
1274         if (action == DELETE ) {
1275             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1276                     " action {}",
1277                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1278         }
1279         if (action == ADD ) {
1280             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1281                     " action {}",
1282                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1283         }
1284
1285         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1286                 rewriteAddress, action);
1287         return status.isSuccess();
1288     }
1289
1290     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1291                                                  String matchAddress, String rewriteAddress,
1292                                                  Action action) {
1293         Status status;
1294         try {
1295             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1296             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1297             status = inboundNatProvider == null ?
1298                     new Status(StatusCode.SUCCESS) :
1299                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1300                             inetMatchAddress, inetRewriteAddress,
1301                             action);
1302         } catch (UnknownHostException e) {
1303             status = new Status(StatusCode.BADREQUEST);
1304         }
1305
1306         if (status.isSuccess()) {
1307             final boolean isSkipped = inboundNatProvider == null;
1308             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1309                     isSkipped ? "skipped" : "programmed",
1310                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1311         } else {
1312             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1313                          " status:{}",
1314                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1315                     status);
1316         }
1317         return status;
1318     }
1319
1320     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1321                                                  Action actionForRewriteExclusion) {
1322         if (actionForRewriteExclusion == DELETE ) {
1323             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1324                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1325         }
1326         if (actionForRewriteExclusion == ADD) {
1327             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1328                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1329         }
1330
1331         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1332     }
1333
1334     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1335                                                    Action actionForNode) {
1336         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1337                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1338
1339         if (status.isSuccess()) {
1340             final boolean isSkipped = outboundNatProvider == null;
1341             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1342                          isSkipped ? "skipped" : "programmed",
1343                          cidr, node.getNodeId().getValue(), actionForNode);
1344         } else {
1345             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1346                          cidr, node.getNodeId().getValue(), actionForNode, status);
1347         }
1348         return status;
1349     }
1350
1351     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1352
1353         if (action == DELETE) {
1354             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1355                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1356         }
1357         if (action == ADD) {
1358             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1359                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1360         }
1361
1362         this.programOutboundIpRewriteStage2(fid, action);
1363     }
1364
1365     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1366         Status status;
1367         try {
1368             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1369             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1370             status = outboundNatProvider == null ?
1371                     new Status(StatusCode.SUCCESS) :
1372                     outboundNatProvider.programIpRewriteRule(
1373                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1374                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1375         } catch (UnknownHostException e) {
1376             status = new Status(StatusCode.BADREQUEST);
1377         }
1378
1379         if (status.isSuccess()) {
1380             final boolean isSkipped = outboundNatProvider == null;
1381             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1382                             " action {}",
1383                          isSkipped ? "skipped" : "programmed",
1384                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1385         } else {
1386             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1387                          " action {} status:{}",
1388                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1389         }
1390         return status;
1391     }
1392
1393     private int getMaskLenFromCidr(String cidr) {
1394         if (cidr == null) {
1395             return 0;
1396         }
1397         String[] splits = cidr.split("/");
1398         if (splits.length != 2) {
1399             return 0;
1400         }
1401
1402         int result;
1403         try {
1404             result = Integer.parseInt(splits[1].trim());
1405         } catch (NumberFormatException nfe) {
1406             result = 0;
1407         }
1408         return result;
1409     }
1410
1411     private Long getDpidForIntegrationBridge(Node node) {
1412         // Check if node is integration bridge; and only then return its dpid
1413         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1414             return southbound.getDataPathId(node);
1415         }
1416         return null;
1417     }
1418
1419     private Long getDpidForExternalBridge(Node node) {
1420         // Check if node is external bridge; and only then return its dpid
1421         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1422             return southbound.getDataPathId(node);
1423         }
1424         return null;
1425     }
1426
1427     private Node getExternalBridgeNode(){
1428         //Pickup the first node that has external bridge (br-ex).
1429         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1430         //the external network is reachable from every br-ex
1431         // TODO: Consider other deployment scenario, and thing of better solution.
1432         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1433         for(Node node : allBridges){
1434             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1435                 return node;
1436             }
1437         }
1438         return null;
1439     }
1440
1441     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1442         if (gatewayPort.getFixedIPs() == null) {
1443             return null;
1444         }
1445         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1446             String subnetUUID = neutronIPs.getSubnetUUID();
1447             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1448             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1449                 return extSubnet;
1450             }
1451             if (extSubnet == null) {
1452                 // TODO: when subnet is created, try again.
1453                 LOG.debug("subnet {} in not found", subnetUUID);
1454              }
1455         }
1456         return null;
1457     }
1458
1459     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1460
1461         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1462         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1463             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1464                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1465                     handleNeutronFloatingIPEvent(neutronFloatingIP, DELETE);
1466                 }
1467             }
1468         }
1469     }
1470
1471     private void updateFloatingIPRules(final NeutronPort neutronPort) {
1472         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1473         if (neutronFloatingIps != null) {
1474             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1475                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1476                     handleNeutronFloatingIPEvent(neutronFloatingIP, UPDATE);
1477                 }
1478             }
1479         }
1480     }
1481
1482
1483     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1484
1485         Preconditions.checkNotNull(gatewayPort);
1486         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1487
1488         if(externalNetwork != null){
1489             if(externalNetwork.isRouterExternal()){
1490                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1491
1492                 // TODO: address IPv6 case.
1493                 if (externalSubnet != null &&
1494                         externalSubnet.getIpVersion() == 4 &&
1495                         gatewayPort.getFixedIPs() != null) {
1496                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1497                     Neutron_IPs neutronIP = null;
1498                     for (Neutron_IPs nIP : gatewayPort.getFixedIPs()) {
1499                         InetAddress ipAddress;
1500                         try {
1501                             ipAddress = InetAddress.getByName(nIP.getIpAddress());
1502                         } catch (UnknownHostException e) {
1503                             LOG.warn("unknown host exception {}", e);
1504                             continue;
1505                         }
1506                         if (ipAddress instanceof Inet4Address) {
1507                             neutronIP = nIP;
1508                             break;
1509                         }
1510                     }
1511                     if (neutronIP == null) {
1512                         // TODO IPv6 neighbor discovery
1513                         LOG.debug("Ignoring gateway ports with IPv6 only fixed ip {}",
1514                                   gatewayPort.getFixedIPs());
1515                     } else {
1516                         gatewayMacResolver.resolveMacAddress(
1517                             this, /* gatewayMacResolverListener */
1518                             null, /* externalNetworkBridgeDpid */
1519                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1520                             new Ipv4Address(externalSubnet.getGatewayIP()),
1521                             new Ipv4Address(neutronIP.getIpAddress()),
1522                             new MacAddress(gatewayPort.getMacAddress()),
1523                             true /* periodicRefresh */);
1524                     }
1525                 } else {
1526                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1527                 }
1528             }
1529         }else{
1530             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1531         }
1532     }
1533
1534
1535     private void storePortInCleanupCache(NeutronPort port) {
1536         this.portCleanupCache.put(port.getPortUUID(),port);
1537     }
1538
1539
1540     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1541         removePortFromCleanupCache(originalPort);
1542         storePortInCleanupCache(updatedPort);
1543     }
1544
1545     public void removePortFromCleanupCache(NeutronPort port) {
1546         if(port != null) {
1547             this.portCleanupCache.remove(port.getPortUUID());
1548         }
1549     }
1550
1551     public Map<String, NeutronPort> getPortCleanupCache() {
1552         return this.portCleanupCache;
1553     }
1554
1555     public NeutronPort getPortFromCleanupCache(String portid) {
1556         for (String neutronPortUuid : this.portCleanupCache.keySet()) {
1557             if (neutronPortUuid.equals(portid)) {
1558                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1559                 return this.portCleanupCache.get(neutronPortUuid);
1560             }
1561         }
1562         return null;
1563     }
1564
1565     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1566         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1567     }
1568
1569
1570     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1571         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1572             if (neutronNetworkUuid.equals(network.getNetworkUUID())) {
1573                 this.networkCleanupCache.remove(neutronNetworkUuid);
1574             }
1575         }
1576         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1577     }
1578
1579     public void removeNetworkFromCleanupCache(String networkid) {
1580         NeutronNetwork network = null;
1581         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1582             if (neutronNetworkUuid.equals(networkid)) {
1583                 network = networkCleanupCache.get(neutronNetworkUuid);
1584                 break;
1585             }
1586         }
1587         if (network != null) {
1588             for (String neutronPortUuid:this.portCleanupCache.keySet()) {
1589                 if (this.portCleanupCache.get(neutronPortUuid).getNetworkUUID().equals(network.getNetworkUUID())) {
1590                     LOG.info("This network is used by another port", network);
1591                     return;
1592                 }
1593             }
1594             this.networkCleanupCache.remove(network.getNetworkUUID());
1595         }
1596     }
1597
1598     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1599         return this.networkCleanupCache;
1600     }
1601
1602     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1603         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1604             if (neutronNetworkUuid.equals(networkid)) {
1605                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", networkid);
1606                 return networkCleanupCache.get(neutronNetworkUuid);
1607             }
1608         }
1609         return null;
1610     }
1611     /**
1612      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1613      *
1614      * @param ofPort the OF port number
1615      * @return the string with encoded OF port (example format "OFPort|999")
1616      */
1617     public static String encodeExcplicitOFPort(Long ofPort) {
1618         return "OFPort|" + ofPort.toString();
1619     }
1620     private void initNetworkCleanUpCache() {
1621         if (this.neutronNetworkCache != null) {
1622             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1623                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1624             }
1625         }
1626     }
1627     private void initPortCleanUpCache() {
1628         if (this.neutronPortCache != null) {
1629             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1630                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1631             }
1632         }
1633     }
1634     @Override
1635     public void setDependencies(ServiceReference serviceReference) {
1636         eventDispatcher =
1637                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1638         eventDispatcher.eventHandlerAdded(serviceReference, this);
1639         tenantNetworkManager =
1640                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1641         configurationService =
1642                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1643         arpProvider =
1644                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1645         inboundNatProvider =
1646                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1647         outboundNatProvider =
1648                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1649         routingProvider =
1650                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1651         l3ForwardingProvider =
1652                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1653         distributedArpService =
1654                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1655         nodeCacheManager =
1656                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1657         southbound =
1658                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1659         gatewayMacResolver =
1660                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1661         securityServicesManager =
1662                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1663
1664         initL3AdapterMembers();
1665     }
1666
1667     @Override
1668     public void setDependencies(Object impl) {
1669         if (impl instanceof INeutronNetworkCRUD) {
1670             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1671             initNetworkCleanUpCache();
1672         } else if (impl instanceof INeutronPortCRUD) {
1673             neutronPortCache = (INeutronPortCRUD)impl;
1674             initPortCleanUpCache();
1675         } else if (impl instanceof INeutronSubnetCRUD) {
1676             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1677         } else if (impl instanceof INeutronFloatingIPCRUD) {
1678             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1679         } else if (impl instanceof ArpProvider) {
1680             arpProvider = (ArpProvider)impl;
1681         } else if (impl instanceof InboundNatProvider) {
1682             inboundNatProvider = (InboundNatProvider)impl;
1683         } else if (impl instanceof OutboundNatProvider) {
1684             outboundNatProvider = (OutboundNatProvider)impl;
1685         } else if (impl instanceof RoutingProvider) {
1686             routingProvider = (RoutingProvider)impl;
1687         } else if (impl instanceof L3ForwardingProvider) {
1688             l3ForwardingProvider = (L3ForwardingProvider)impl;
1689         }else if (impl instanceof GatewayMacResolver) {
1690             gatewayMacResolver = (GatewayMacResolver)impl;
1691         }else if (impl instanceof IcmpEchoProvider) {
1692             icmpEchoProvider = (IcmpEchoProvider)impl;
1693         }
1694
1695         populateL3ForwardingCaches();
1696     }
1697 }