openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  15 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  16 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  17 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  18 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  19 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.IcmpEchoProvider;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  37 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  38 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  39 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  49 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  50 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  60 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  61 import org.osgi.framework.ServiceReference;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 import java.net.Inet4Address;
  66 import java.net.InetAddress;
  67 import java.net.UnknownHostException;
  68 import java.util.ArrayList;
  69 import java.util.Collection;
  70 import java.util.HashMap;
  71 import java.util.HashSet;
  72 import java.util.Iterator;
  73 import java.util.List;
  74 import java.util.Map;
  75 import java.util.Set;
  76
  77 /**
  78  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  79  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  80  * as well as the multi-tenant router forwarding provider.
  81  */
  82 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  83     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  84
  85     // The implementation for each of these services is resolved by the OSGi Service Manager
  86     private volatile ConfigurationService configurationService;
  87     private volatile TenantNetworkManager tenantNetworkManager;
  88     private volatile NodeCacheManager nodeCacheManager;
  89     private volatile INeutronNetworkCRUD neutronNetworkCache;
  90     private volatile INeutronSubnetCRUD neutronSubnetCache;
  91     private volatile INeutronPortCRUD neutronPortCache;
  92     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  93     private volatile L3ForwardingProvider l3ForwardingProvider;
  94     private volatile InboundNatProvider inboundNatProvider;
  95     private volatile OutboundNatProvider outboundNatProvider;
  96     private volatile ArpProvider arpProvider;
  97     private volatile RoutingProvider routingProvider;
  98     private volatile GatewayMacResolver gatewayMacResolver;
  99     private volatile SecurityServicesManager securityServicesManager;
 100     private volatile IcmpEchoProvider icmpEchoProvider;
 101
 102     private class FloatIpData {
 103         // br-int of node where floating ip is associated with tenant port
 104         private final Long dpid;
 105         // patch port in br-int used to reach br-ex
 106         private final Long ofPort;
 107         // segmentation id of the net where fixed ip is instantiated
 108         private final String segId;
 109         // mac address assigned to neutron port of floating ip
 110         private final String macAddress;
 111         private final String floatingIpAddress;
 112         // ip address given to tenant vm
 113         private final String fixedIpAddress;
 114         private final String neutronRouterMac;
 115
 116         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 117                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 118             this.dpid = dpid;
 119             this.ofPort = ofPort;
 120             this.segId = segId;
 121             this.macAddress = macAddress;
 122             this.floatingIpAddress = floatingIpAddress;
 123             this.fixedIpAddress = fixedIpAddress;
 124             this.neutronRouterMac = neutronRouterMac;
 125         }
 126     }
 127
 128     private Map<String, String> networkIdToRouterMacCache;
 129     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 130     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 131
 132     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 133     private Map<String, FloatIpData> floatIpDataMapCache;
 134
 135     private String externalRouterMac;
 136     private Boolean enabled = false;
 137     private Boolean isCachePopulationDone = false;
 138     private Map<String, NeutronPort> portCleanupCache;
 139     private Map<String, NeutronNetwork> networkCleanupCache;
 140
 141     private Southbound southbound;
 142     private DistributedArpService distributedArpService;
 143     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 144
 145     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 146     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 147     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 148     private static final String OWNER_FLOATING_IP = "network:floatingip";
 149     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 150
 151     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 152         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 153         this.neutronModelsDataStoreHelper = neutronHelper;
 154     }
 155
 156     private void initL3AdapterMembers() {
 157         Preconditions.checkNotNull(configurationService);
 158
 159         if (configurationService.isL3ForwardingEnabled()) {
 160             this.networkIdToRouterMacCache = new HashMap<>();
 161             this.networkIdToRouterIpListCache = new HashMap<>();
 162             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 163             this.neutronPortToDpIdCache = new HashMap<>();
 164             this.floatIpDataMapCache = new HashMap<>();
 165
 166             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 167             if (this.externalRouterMac == null) {
 168                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 169             }
 170             this.enabled = true;
 171             LOG.info("OVSDB L3 forwarding is enabled");
 172         } else {
 173             LOG.debug("OVSDB L3 forwarding is disabled");
 174         }
 175         this.portCleanupCache = new HashMap<>();
 176         this.networkCleanupCache = new HashMap<>();
 177     }
 178
 179     //
 180     // Callbacks from AbstractHandler
 181     //
 182     @Override
 183     public void processEvent(AbstractEvent abstractEvent) {
 184         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 185             LOG.error("Unable to process abstract event " + abstractEvent);
 186             return;
 187         }
 188         if (!this.enabled) {
 189             return;
 190         }
 191
 192         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 193         switch (ev.getAction()) {
 194             case UPDATE:
 195                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 196                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 197                 } else {
 198                     LOG.warn("Received update for an unexpected event " + ev);
 199                 }
 200                 break;
 201             case ADD:
 202                 // fall through...
 203                 // break;
 204             case DELETE:
 205                 // fall through...
 206                 // break;
 207             default:
 208                 LOG.warn("Unable to process event " + ev);
 209                 break;
 210         }
 211     }
 212
 213     //
 214     // Callbacks from GatewayMacResolverListener
 215     //
 216
 217     @Override
 218     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 219         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 220                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 221         if (!this.enabled) {
 222             return;
 223         }
 224
 225         if (macAddress == null || macAddress.getValue() == null) {
 226             // TODO: handle cases when mac is null
 227             return;
 228         }
 229
 230         //
 231         // Enqueue event so update is handled by adapter's thread
 232         //
 233         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 234     }
 235
 236     private void populateL3ForwardingCaches() {
 237         if (!this.enabled) {
 238             return;
 239         }
 240         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 241                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 242             return;
 243         }
 244         this.isCachePopulationDone = true;
 245         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 246         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 247         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 248         if(routers != null && routers.getRouter() != null && ports != null) {
 249             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 250             for( Router router : routers.getRouter()) {
 251                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 252                 if(!ports.getPort().isEmpty()) {
 253                     for( Port port : ports.getPort()) {
 254                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 255                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 256                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 257                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 258                                     , port.getMacAddress());
 259
 260                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 261                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 262                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 263                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 264                         }
 265                     }
 266                 }else {
 267                     LOG.warn("L3 Cache Population :Did not find any port information " +
 268                             "in config Data Store for router {}",router);
 269                 }
 270             }
 271         }
 272         LOG.debug("NetVirt L3 caches population is done");
 273     }
 274
 275     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 276         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 277             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 278             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 279             for(Node bridge : bridges) {
 280                 List<OvsdbTerminationPointAugmentation> interfaces
 281                         = southbound.extractTerminationPointAugmentations(bridge);
 282                 if(interfaces != null && !interfaces.isEmpty()) {
 283                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 284                             interfaces.size(), bridge.getNodeId());
 285                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 286                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 287                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 288                             Long dpId = getDpidForIntegrationBridge(bridge);
 289                             Uuid interfaceUuid = intf.getInterfaceUuid();
 290                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 291                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 292                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 293                             break;
 294                         }
 295                     }
 296                 }
 297             }
 298         }
 299         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 300     }
 301
 302     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 303         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 304         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 305         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 306             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 307                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 308                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 309                             "Fetching from data store.",neutronFloatingIP.getID());
 310                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 311                 }
 312             }
 313         }
 314         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 315         return floatIpDataMapCache.values();
 316     }
 317     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 318         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 319                 "floating id {} ",neutronFloatingId);
 320         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 321             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 322             if (neutronFloatingIP == null) {
 323                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 324                 return null;
 325             }
 326             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 327             NeutronPort neutronPortForFloatIp = null;
 328             for (NeutronPort neutronPort : neutronPorts) {
 329                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 330                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 331                     neutronPortForFloatIp = neutronPort;
 332                     break;
 333                 }
 334             }
 335
 336             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 337             if(neutronTenantPortUuid == null) {
 338                 return null;
 339             }
 340             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 341             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 342             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 343             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 344
 345             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 346             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 347                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 348             String providerSegmentationId = tenantNeutronNetwork != null ?
 349                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 350             String neutronRouterMac = tenantNeutronNetwork != null ?
 351                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 352
 353             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 354                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 355                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 356                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 357                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 358                                 "seg {} mac {} rtrMac {}",
 359                         fixedIpAddress,
 360                         floatingIpAddress,
 361                         neutronPortForFloatIp,
 362                         neutronTenantPortUuid,
 363                         providerSegmentationId,
 364                         floatingIpMac,
 365                         neutronRouterMac);
 366
 367                 return null;
 368             }
 369
 370             // get ofport for patch port in br-int
 371             final Long dpId = nodeIfPair.getLeft();
 372             final Long ofPort = findOFPortForExtPatch(dpId);
 373             if (ofPort == null) {
 374                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 375                                 "to connect floating ip to external bridge. dpid {}",
 376                         dpId);
 377                 return null;
 378             }
 379
 380             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 381                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 382             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 383
 384         }
 385         return floatIpDataMapCache.get(neutronFloatingId);
 386     }
 387     /**
 388      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 389      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 390      * subscribe so that the mac address associated to the gateway ip address is available.
 391      *
 392      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 393      */
 394     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 395         Preconditions.checkNotNull(externalRouterMacUpdate);
 396
 397         flushExistingIpRewrite();
 398         this.externalRouterMac = externalRouterMacUpdate;
 399         rebuildExistingIpRewrite();
 400     }
 401
 402     /**
 403      * Process the event.
 404      *
 405      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 406      * @param subnet An instance of NeutronSubnet object.
 407      */
 408     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 409         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 410         if (action == Action.ADD) {
 411             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 412         }
 413     }
 414
 415     /**
 416      * Process the port event as a router interface event.
 417      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 418      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 419      *
 420      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 421      * @param neutronPort An instance of NeutronPort object.
 422      */
 423     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 424         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 425
 426         if (action == Action.UPDATE) {
 427             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 428             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 429             this.processSecurityGroupUpdate(neutronPort);
 430         }
 431
 432         if (!this.enabled) {
 433             return;
 434         }
 435
 436         final boolean isDelete = action == Action.DELETE;
 437
 438         if (action == Action.DELETE) {
 439             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 440             this.cleanupFloatingIPRules(neutronPort);
 441         }
 442
 443         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 444             if (!isDelete) {
 445                 LOG.info("Port {} is network router gateway interface, "
 446                         + "triggering gateway resolution for the attached external network", neutronPort);
 447                 this.triggerGatewayMacResolver(neutronPort);
 448             }else{
 449                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 450                 if (null == externalNetwork) {
 451                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 452                 }
 453
 454                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 455                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 456                     // TODO support IPv6
 457                     if (externalSubnet != null &&
 458                             externalSubnet.getIpVersion() == 4) {
 459                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 460                     }
 461                 }
 462             }
 463         }
 464
 465         // Treat the port event as a router interface event if the port belongs to router. This is a
 466         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 467         //
 468         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 469             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 470
 471             if (neutronPort.getFixedIPs() != null) {
 472                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 473                     NeutronRouter_Interface neutronRouterInterface =
 474                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 475                     // id of router interface to be same as subnet
 476                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 477                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 478
 479                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 480                 }
 481             }
 482         } else {
 483             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 484             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 485             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 486             // there.
 487             //
 488             if (!isDelete && neutronPort.getFixedIPs() != null) {
 489                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 490                     NeutronRouter_Interface neutronRouterInterface =
 491                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 492                     if (neutronRouterInterface != null) {
 493                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 494                     }
 495                 }
 496             }
 497             this.updateL3ForNeutronPort(neutronPort, isDelete);
 498         }
 499     }
 500
 501     /**
 502      * Process the event.
 503      *
 504      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 505      * @param neutronRouter An instance of NeutronRouter object.
 506      */
 507     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 508         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 509     }
 510
 511     /**
 512      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 513      * delete the ports on the same subnet.
 514      *
 515      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 516      * @param neutronRouter An instance of NeutronRouter object.
 517      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 518      */
 519     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 520                                                   final NeutronRouter_Interface neutronRouterInterface,
 521                                                   Action action) {
 522         LOG.debug("Router interface {} got event {}. Subnet {}",
 523                      neutronRouterInterface.getPortUUID(),
 524                      action,
 525                      neutronRouterInterface.getSubnetUUID());
 526         if (!this.enabled) {
 527             return;
 528         }
 529
 530         final boolean isDelete = action == Action.DELETE;
 531
 532         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 533
 534         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 535         // see if they are affected by l3
 536         //
 537         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 538             boolean currPortShouldBeDeleted = false;
 539             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 540             if (isDelete) {
 541                 if (neutronPort.getFixedIPs() != null) {
 542                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 543                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 544                             currPortShouldBeDeleted = true;
 545                             break;
 546                         }
 547                     }
 548                 }
 549             }
 550             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 551         }
 552
 553     }
 554
 555     /**
 556      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 557      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 558      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 559      * way around) on OpenFlow Table 100.
 560      *
 561      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 562      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 563      */
 564     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 565                                              Action actionIn) {
 566         Preconditions.checkNotNull(neutronFloatingIP);
 567
 568         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 569                 neutronFloatingIP.getFixedIPAddress(),
 570                 neutronFloatingIP.getFloatingIPAddress(),
 571                 neutronFloatingIP.getFloatingNetworkUUID());
 572         if (!this.enabled) {
 573             return;
 574         }
 575
 576         Action action;
 577
 578         // Consider action to be delete if getFixedIPAddress is null
 579         //
 580         if (neutronFloatingIP.getFixedIPAddress() == null) {
 581             action = Action.DELETE;
 582         } else {
 583             action = actionIn;
 584         }
 585
 586         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 587
 588         if (action != Action.DELETE) {
 589             // must be first, as it updates floatIpDataMapCache
 590             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 591
 592             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 593             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 594         } else {
 595             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 596             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 597
 598             // must be last, as it updates floatIpDataMapCache
 599             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 600         }
 601     }
 602
 603     /**
 604      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 605      * ip, otherwise for newer one.
 606      */
 607     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 608         Preconditions.checkNotNull(neutronFloatingIP);
 609
 610         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 611         if (fid == null) {
 612             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 613                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 614             return;
 615         }
 616         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 617                                       action);
 618     }
 619
 620     /**
 621      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 622      * ip, otherwise for newer one.
 623      */
 624     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 625         Preconditions.checkNotNull(neutronFloatingIP);
 626
 627         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 628         if (fid == null) {
 629             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 630                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 631             return;
 632         }
 633         programOutboundIpRewriteStage1(fid, action);
 634     }
 635
 636     private void flushExistingIpRewrite() {
 637         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 638             programOutboundIpRewriteStage1(fid, Action.DELETE);
 639         }
 640     }
 641
 642     private void rebuildExistingIpRewrite() {
 643         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 644             programOutboundIpRewriteStage1(fid, Action.ADD);
 645         }
 646     }
 647
 648     /**
 649      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 650      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 651      * requests.
 652      */
 653     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 654         Preconditions.checkNotNull(neutronFloatingIP);
 655         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 656         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 657
 658         // find bridge Node where floating ip is configured by looking up cache for its port
 659         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 660         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 661         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 662         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 663         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 664         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 665
 666         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 667         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 668                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 669         final String providerSegmentationId = tenantNeutronNetwork != null ?
 670                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 671         final String neutronRouterMac = tenantNeutronNetwork != null ?
 672                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 673
 674         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 675                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 676                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 677                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 678             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 679                     fixedIpAddress,
 680                     floatingIpAddress,
 681                     neutronPortForFloatIp,
 682                     neutronTenantPortUuid,
 683                     providerSegmentationId,
 684                     floatingIpMac,
 685                     neutronRouterMac);
 686             return;
 687         }
 688
 689         // get ofport for patch port in br-int
 690         final Long dpId = nodeIfPair.getLeft();
 691         final Long ofPort = findOFPortForExtPatch(dpId);
 692         if (ofPort == null) {
 693             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 694                     dpId);
 695             return;
 696         }
 697
 698         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 699         //
 700         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 701                 Action.ADD)) {
 702             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 703                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 704             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 705             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 706                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 707                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 708         }
 709     }
 710
 711     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 712         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 713         if (floatIpData == null) {
 714             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 715             return;
 716         }
 717
 718         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 719                 floatIpData.floatingIpAddress, Action.DELETE)) {
 720             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 721             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 722                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 723         }
 724     }
 725
 726     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 727         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 728             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 729                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 730                 return neutronPort;
 731             }
 732         }
 733         return null;
 734     }
 735
 736     private Long findOFPortForExtPatch(Long dpId) {
 737         final String brInt = configurationService.getIntegrationBridgeName();
 738         final String brExt = configurationService.getExternalBridgeName();
 739         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 740
 741         Preconditions.checkNotNull(dpId);
 742         Preconditions.checkNotNull(portNameInt);
 743
 744         final long dpidPrimitive = dpId;
 745         for (Node node : nodeCacheManager.getBridgeNodes()) {
 746             if (dpidPrimitive == southbound.getDataPathId(node)) {
 747                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 748                         southbound.getTerminationPointOfBridge(node, portNameInt);
 749                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 750             }
 751         }
 752         return null;
 753     }
 754
 755     /**
 756      * Process the event.
 757      *
 758      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 759      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 760      */
 761     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 762         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 763         if (action == Action.UPDATE) {
 764             this.updateNetworkInCleanupCache(neutronNetwork);
 765         }
 766     }
 767
 768     //
 769     // Callbacks from OVSDB's southbound handler
 770     //
 771     /**
 772      * Process the event.
 773      *
 774      * @param bridgeNode An instance of Node object.
 775      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 776      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 777      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 778      * object.
 779      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 780      */
 781     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 782                                      final NeutronNetwork neutronNetwork, Action action) {
 783         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 784                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 785
 786         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 787         if (action != Action.DELETE && neutronPort != null) {
 788             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 789             storePortInCleanupCache(neutronPort);
 790         }
 791
 792         if (!this.enabled) {
 793             return;
 794         }
 795
 796         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 797         final Uuid interfaceUuid = intf.getInterfaceUuid();
 798
 799         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 800                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 801
 802         if (neutronPort != null) {
 803             final String neutronPortUuid = neutronPort.getPortUUID();
 804
 805             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 806                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 807             }
 808
 809             handleNeutronPortEvent(neutronPort, action);
 810         }
 811
 812         if (action == Action.DELETE && interfaceUuid != null) {
 813             handleInterfaceEventDelete(intf, dpId);
 814         }
 815     }
 816
 817     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 818         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 819         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 820                 neutronPortUuid, dpId, interfaceUuid.getValue());
 821     }
 822
 823     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 824         // Remove entry from neutronPortToDpIdCache based on interface uuid
 825         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 826             final String currPortUuid = entry.getKey();
 827             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 828                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 829                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 830                 neutronPortToDpIdCache.remove(currPortUuid);
 831                 break;
 832             }
 833         }
 834     }
 835
 836     //
 837     // Internal helpers
 838     //
 839     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 840
 841         final String networkUUID = neutronPort.getNetworkUUID();
 842         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 843
 844         if(!isDelete) {
 845             // If there is no router interface handling the networkUUID, we are done
 846             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 847                 return;
 848             }
 849
 850             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 851             // router interface are handled via handleNeutronRouterInterfaceEvent.
 852             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 853                 return;
 854             }
 855         }
 856
 857         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 858         final String providerSegmentationId = neutronNetwork != null ?
 859                                               neutronNetwork.getProviderSegmentationID() : null;
 860         final String tenantMac = neutronPort.getMacAddress();
 861
 862         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 863             tenantMac == null || tenantMac.isEmpty()) {
 864             // done: go no further w/out all the info needed...
 865             return;
 866         }
 867
 868         final Action action = isDelete ? Action.DELETE : Action.ADD;
 869         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 870         if (nodes.isEmpty()) {
 871             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 872         }
 873         for (Node node : nodes) {
 874             final Long dpid = getDpidForIntegrationBridge(node);
 875             if (dpid == null) {
 876                 continue;
 877             }
 878             if (neutronPort.getFixedIPs() == null) {
 879                 continue;
 880             }
 881             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 882                 final String tenantIpStr = neutronIP.getIpAddress();
 883                 if (tenantIpStr.isEmpty()) {
 884                     continue;
 885                 }
 886
 887                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 888                 // still needed when routing to subnets non-local to node (bug 2076).
 889                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 890             }
 891         }
 892     }
 893
 894     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 895         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 896         /**
 897          * Get updated data and original data for the the changed. Identify the security groups that got
 898          * added and removed and call the appropriate providers for updating the flows.
 899          */
 900         try {
 901             NeutronPort originalPort = neutronPort.getOriginalPort();
 902             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 903                                                                             neutronPort.getOriginalPort());
 904             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 905                                                                               neutronPort);
 906
 907             if (null != addedGroup && !addedGroup.isEmpty()) {
 908                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 909             }
 910             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 911                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 912             }
 913
 914         } catch (Exception e) {
 915             LOG.error("Exception in processSecurityGroupUpdate", e);
 916         }
 917     }
 918
 919     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 920         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 921         if (port1 == null) {
 922             return null;
 923         }
 924         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 925         if (port2 == null) {
 926             return list1;
 927         }
 928         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 929         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 930             NeutronSecurityGroup securityGroup1 = iterator.next();
 931             for (NeutronSecurityGroup securityGroup2 :list2) {
 932                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 933                     iterator.remove();
 934                 }
 935             }
 936         }
 937         return list1;
 938     }
 939
 940     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 941                                            String macAddress, String ipStr,
 942                                            Action actionForNode) {
 943         if (actionForNode == Action.DELETE) {
 944             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 945                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 946         }
 947         if (actionForNode == Action.ADD) {
 948             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 949                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 950         }
 951
 952         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 953                                                        macAddress, ipStr, actionForNode);
 954     }
 955
 956     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 957                                              String macAddress,
 958                                              String address,
 959                                              Action actionForNode) {
 960         Status status;
 961         try {
 962             InetAddress inetAddress = InetAddress.getByName(address);
 963             status = l3ForwardingProvider == null ?
 964                      new Status(StatusCode.SUCCESS) :
 965                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 966                                                                       inetAddress, macAddress, actionForNode);
 967         } catch (UnknownHostException e) {
 968             status = new Status(StatusCode.BADREQUEST);
 969         }
 970
 971         if (status.isSuccess()) {
 972             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 973                          l3ForwardingProvider == null ? "skipped" : "programmed",
 974                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 975         } else {
 976             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 977                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 978         }
 979         return status;
 980     }
 981
 982     // --
 983
 984     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 985                                                        Boolean isDelete) {
 986         Preconditions.checkNotNull(destNeutronRouterInterface);
 987
 988         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 989         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 990         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 991         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 992         final NeutronNetwork neutronNetwork = subnet != null ?
 993                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 994         final String destinationSegmentationId = neutronNetwork != null ?
 995                                                  neutronNetwork.getProviderSegmentationID() : null;
 996         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 997         final String cidr = subnet != null ? subnet.getCidr() : null;
 998         final int mask = getMaskLenFromCidr(cidr);
 999
1000         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1001                      destNeutronRouterInterface, isDelete);
1002
1003         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1004         // the local cache
1005         if (neutronNetwork != null) {
1006             if (macAddress == null || macAddress.isEmpty()) {
1007                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1008             }
1009             if (ipList == null || ipList.isEmpty()) {
1010                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1011             }
1012         }
1013
1014         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1015             cidr == null || cidr.isEmpty() ||
1016             macAddress == null || macAddress.isEmpty() ||
1017             ipList == null || ipList.isEmpty()) {
1018             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1019                          destinationSegmentationId, cidr, macAddress, ipList);
1020             // done: go no further w/out all the info needed...
1021             return;
1022         }
1023
1024         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1025
1026         // Keep cache for finding router's mac from network uuid -- add
1027         //
1028         if (! isDelete) {
1029             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1030             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1031             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1032         }
1033
1034         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1035         if (nodes.isEmpty()) {
1036             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1037         }
1038         for (Node node : nodes) {
1039             final Long dpid = getDpidForIntegrationBridge(node);
1040             if (dpid == null) {
1041                 continue;
1042             }
1043
1044             for (Neutron_IPs neutronIP : ipList) {
1045                 final String ipStr = neutronIP.getIpAddress();
1046                 if (ipStr.isEmpty()) {
1047                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1048                             node.getNodeId().getValue(), ipStr);
1049                     continue;
1050                 }
1051
1052                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1053                 //
1054                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1055                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1056                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1057                                                               neutronNetwork, destinationSegmentationId,
1058                                                               macAddress, ipStr, mask, actionForNode,
1059                                                               true /*isReflexsive*/);
1060                 }
1061
1062                 if (! isExternal) {
1063                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1064                             actionForNode);
1065                 }
1066                 // Enable ARP responder by default, because router interface needs to be responded always.
1067                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1068                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1069             }
1070
1071             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1072             // for the external neutron networks.
1073             //
1074             {
1075                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1076                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1077             }
1078         }
1079
1080         if (isDelete) {
1081             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1082             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1083             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1084         }
1085     }
1086
1087     private void programFlowForNetworkFromExternal(final Node node,
1088                                                    final Long dpid,
1089                                                    final String destinationSegmentationId,
1090                                                    final String dstMacAddress,
1091                                                    final String destIpStr,
1092                                                    final int destMask,
1093                                                    final Action actionForNode) {
1094         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1095                 dstMacAddress, destIpStr, destMask, actionForNode);
1096     }
1097
1098     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1099                                                            final Long dpid,
1100                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1101                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1102                                                            final NeutronNetwork dstNeutronNetwork,
1103                                                            final String destinationSegmentationId,
1104                                                            final String dstMacAddress,
1105                                                            final String destIpStr,
1106                                                            final int destMask,
1107                                                            final Action actionForNode,
1108                                                            Boolean isReflexsive) {
1109         Preconditions.checkNotNull(srcNeutronRouterInterface);
1110         Preconditions.checkNotNull(dstNeutronRouterInterface);
1111
1112         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1113         if (sourceSubnetId == null) {
1114             LOG.error("Could not get provider Subnet ID from router interface {}",
1115                          srcNeutronRouterInterface.getID());
1116             return;
1117         }
1118
1119         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1120         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1121         if (sourceNetworkId == null) {
1122             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1123             return;
1124         }
1125
1126         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1127         if (sourceNetwork == null) {
1128             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1129             return;
1130         }
1131
1132         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1133             // Isolate subnets from different tenants within the same router
1134             return;
1135         }
1136         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1137         if (sourceSegmentationId == null) {
1138             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1139             return;
1140         }
1141         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1142             // Skip 'self'
1143             return;
1144         }
1145
1146         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1147                                      dstMacAddress, destIpStr, destMask, actionForNode);
1148
1149         // Flip roles src->dst; dst->src
1150         if (isReflexsive) {
1151             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1152             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1153             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1154             final String cidr2 = sourceSubnet.getCidr();
1155             final int mask2 = getMaskLenFromCidr(cidr2);
1156
1157             if (cidr2 == null || cidr2.isEmpty() ||
1158                 macAddress2 == null || macAddress2.isEmpty() ||
1159                 ipList2 == null || ipList2.isEmpty()) {
1160                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1161                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1162                 // done: go no further w/out all the info needed...
1163                 return;
1164             }
1165
1166             for (Neutron_IPs neutronIP2 : ipList2) {
1167                 final String ipStr2 = neutronIP2.getIpAddress();
1168                 if (ipStr2.isEmpty()) {
1169                     continue;
1170                 }
1171                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1172                                                           srcNeutronRouterInterface,
1173                                                           sourceNetwork, sourceSegmentationId,
1174                                                           macAddress2, ipStr2, mask2, actionForNode,
1175                                                           false /*isReflexsive*/);
1176             }
1177         }
1178     }
1179
1180     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1181                                               String destinationSegmentationId,
1182                                               String macAddress, String ipStr, int mask,
1183                                               Action actionForNode) {
1184         if (actionForNode == Action.DELETE) {
1185             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1186                          " action {}",
1187                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1188                          macAddress, ipStr, mask, actionForNode);
1189         }
1190         if (actionForNode == Action.ADD) {
1191             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1192                          " action {}",
1193                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1194                          macAddress, ipStr, mask, actionForNode);
1195         }
1196
1197         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1198                                                           macAddress, ipStr, mask, actionForNode);
1199     }
1200
1201     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1202                                                 String destinationSegmentationId,
1203                                                 String macAddress,
1204                                                 String address, int mask,
1205                                                 Action actionForNode) {
1206         Status status;
1207         try {
1208             InetAddress inetAddress = InetAddress.getByName(address);
1209             status = routingProvider == null ?
1210                      new Status(StatusCode.SUCCESS) :
1211                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1212                                                             macAddress, inetAddress, mask, actionForNode);
1213         } catch (UnknownHostException e) {
1214             status = new Status(StatusCode.BADREQUEST);
1215         }
1216
1217         if (status.isSuccess()) {
1218             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1219                          routingProvider == null ? "skipped" : "programmed",
1220                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1221                          actionForNode);
1222         } else {
1223             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1224                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1225                          actionForNode, status);
1226         }
1227         return status;
1228     }
1229
1230     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1231                                            String macAddress, String ipStr,
1232                                            Action action) {
1233         if (action == Action.DELETE ) {
1234             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1235                     dpid, segOrOfPort, macAddress, ipStr, action);
1236         }
1237         if (action == Action.ADD) {
1238             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1239                     dpid, segOrOfPort, macAddress, ipStr, action);
1240         }
1241
1242         Status status = new Status(StatusCode.UNSUPPORTED);
1243         if (icmpEchoProvider != null){
1244             try {
1245                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1246                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1247                                                 macAddress, inetAddress, action);
1248             } catch (UnknownHostException e) {
1249                 status = new Status(StatusCode.BADREQUEST);
1250             }
1251         }
1252
1253         if (status.isSuccess()) {
1254             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1255                     icmpEchoProvider == null ? "skipped" : "programmed",
1256                     macAddress, ipStr, dpid, segOrOfPort, action);
1257         } else {
1258             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1259                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1260         }
1261
1262         return status.isSuccess();
1263     }
1264
1265     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1266                                                   String matchAddress, String rewriteAddress,
1267                                                   Action action) {
1268         if (action == Action.DELETE ) {
1269             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1270                     " action {}",
1271                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1272         }
1273         if (action == Action.ADD ) {
1274             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1275                     " action {}",
1276                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1277         }
1278
1279         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1280                 rewriteAddress, action);
1281         return status.isSuccess();
1282     }
1283
1284     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1285                                                  String matchAddress, String rewriteAddress,
1286                                                  Action action) {
1287         Status status;
1288         try {
1289             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1290             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1291             status = inboundNatProvider == null ?
1292                     new Status(StatusCode.SUCCESS) :
1293                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1294                             inetMatchAddress, inetRewriteAddress,
1295                             action);
1296         } catch (UnknownHostException e) {
1297             status = new Status(StatusCode.BADREQUEST);
1298         }
1299
1300         if (status.isSuccess()) {
1301             final boolean isSkipped = inboundNatProvider == null;
1302             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1303                     isSkipped ? "skipped" : "programmed",
1304                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1305         } else {
1306             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1307                          " status:{}",
1308                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1309                     status);
1310         }
1311         return status;
1312     }
1313
1314     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1315                                                  Action actionForRewriteExclusion) {
1316         if (actionForRewriteExclusion == Action.DELETE ) {
1317             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1318                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1319         }
1320         if (actionForRewriteExclusion == Action.ADD) {
1321             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1322                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1323         }
1324
1325         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1326     }
1327
1328     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1329                                                    Action actionForNode) {
1330         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1331                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1332
1333         if (status.isSuccess()) {
1334             final boolean isSkipped = outboundNatProvider == null;
1335             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1336                          isSkipped ? "skipped" : "programmed",
1337                          cidr, node.getNodeId().getValue(), actionForNode);
1338         } else {
1339             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1340                          cidr, node.getNodeId().getValue(), actionForNode, status);
1341         }
1342         return status;
1343     }
1344
1345     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1346
1347         if (action == Action.DELETE) {
1348             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1349                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1350         }
1351         if (action == Action.ADD) {
1352             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1353                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1354         }
1355
1356         this.programOutboundIpRewriteStage2(fid, action);
1357     }
1358
1359     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1360         Status status;
1361         try {
1362             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1363             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1364             status = outboundNatProvider == null ?
1365                     new Status(StatusCode.SUCCESS) :
1366                     outboundNatProvider.programIpRewriteRule(
1367                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1368                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1369         } catch (UnknownHostException e) {
1370             status = new Status(StatusCode.BADREQUEST);
1371         }
1372
1373         if (status.isSuccess()) {
1374             final boolean isSkipped = outboundNatProvider == null;
1375             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1376                             " action {}",
1377                          isSkipped ? "skipped" : "programmed",
1378                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1379         } else {
1380             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1381                          " action {} status:{}",
1382                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1383         }
1384         return status;
1385     }
1386
1387     private int getMaskLenFromCidr(String cidr) {
1388         if (cidr == null) {
1389             return 0;
1390         }
1391         String[] splits = cidr.split("/");
1392         if (splits.length != 2) {
1393             return 0;
1394         }
1395
1396         int result;
1397         try {
1398             result = Integer.parseInt(splits[1].trim());
1399         } catch (NumberFormatException nfe) {
1400             result = 0;
1401         }
1402         return result;
1403     }
1404
1405     private Long getDpidForIntegrationBridge(Node node) {
1406         // Check if node is integration bridge; and only then return its dpid
1407         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1408             return southbound.getDataPathId(node);
1409         }
1410         return null;
1411     }
1412
1413     private Long getDpidForExternalBridge(Node node) {
1414         // Check if node is external bridge; and only then return its dpid
1415         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1416             return southbound.getDataPathId(node);
1417         }
1418         return null;
1419     }
1420
1421     private Node getExternalBridgeNode(){
1422         //Pickup the first node that has external bridge (br-ex).
1423         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1424         //the external network is reachable from every br-ex
1425         // TODO: Consider other deployment scenario, and thing of better solution.
1426         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1427         for(Node node : allBridges){
1428             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1429                 return node;
1430             }
1431         }
1432         return null;
1433     }
1434
1435     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1436         if (gatewayPort.getFixedIPs() == null) {
1437             return null;
1438         }
1439         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1440             String subnetUUID = neutronIPs.getSubnetUUID();
1441             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1442             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1443                 return extSubnet;
1444             }
1445             if (extSubnet == null) {
1446                 // TODO: when subnet is created, try again.
1447                 LOG.debug("subnet {} in not found", subnetUUID);
1448              }
1449         }
1450         return null;
1451     }
1452
1453     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1454
1455         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1456         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1457             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1458                 if (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID())) {
1459                     handleNeutronFloatingIPEvent(neutronFloatingIP, Action.DELETE);
1460                 }
1461             }
1462         }
1463     }
1464
1465
1466     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1467
1468         Preconditions.checkNotNull(gatewayPort);
1469         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1470
1471         if(externalNetwork != null){
1472             if(externalNetwork.isRouterExternal()){
1473                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1474
1475                 // TODO: address IPv6 case.
1476                 if (externalSubnet != null &&
1477                         externalSubnet.getIpVersion() == 4 &&
1478                         gatewayPort.getFixedIPs() != null) {
1479                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1480                     Neutron_IPs neutronIP = null;
1481                     for (Neutron_IPs nIP : gatewayPort.getFixedIPs()) {
1482                         InetAddress ipAddress;
1483                         try {
1484                             ipAddress = InetAddress.getByAddress(nIP.getIpAddress().getBytes());
1485                         } catch (UnknownHostException e) {
1486                             LOG.warn("unknown host exception {}", e);
1487                             continue;
1488                         }
1489                         if (ipAddress instanceof Inet4Address) {
1490                             neutronIP = nIP;
1491                             break;
1492                         }
1493                     }
1494                     if (neutronIP == null) {
1495                         // TODO IPv6 neighbor discovery
1496                         LOG.debug("Ignoring gateway ports with IPv6 only fixed ip {}",
1497                                   gatewayPort.getFixedIPs());
1498                     } else {
1499                         gatewayMacResolver.resolveMacAddress(
1500                             this, /* gatewayMacResolverListener */
1501                             null, /* externalNetworkBridgeDpid */
1502                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1503                             new Ipv4Address(externalSubnet.getGatewayIP()),
1504                             new Ipv4Address(neutronIP.getIpAddress()),
1505                             new MacAddress(gatewayPort.getMacAddress()),
1506                             true /* periodicRefresh */);
1507                     }
1508                 } else {
1509                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1510                 }
1511             }
1512         }else{
1513             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1514         }
1515     }
1516
1517
1518     private void storePortInCleanupCache(NeutronPort port) {
1519         this.portCleanupCache.put(port.getPortUUID(),port);
1520     }
1521
1522
1523     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1524         removePortFromCleanupCache(originalPort);
1525         storePortInCleanupCache(updatedPort);
1526     }
1527
1528     public void removePortFromCleanupCache(NeutronPort port) {
1529         if(port != null) {
1530             this.portCleanupCache.remove(port.getPortUUID());
1531         }
1532     }
1533
1534     public Map<String, NeutronPort> getPortCleanupCache() {
1535         return this.portCleanupCache;
1536     }
1537
1538     public NeutronPort getPortFromCleanupCache(String portid) {
1539         for (String neutronPortUuid : this.portCleanupCache.keySet()) {
1540             if (neutronPortUuid.equals(portid)) {
1541                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1542                 return this.portCleanupCache.get(neutronPortUuid);
1543             }
1544         }
1545         return null;
1546     }
1547
1548     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1549         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1550     }
1551
1552
1553     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1554         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1555             if (neutronNetworkUuid.equals(network.getNetworkUUID())) {
1556                 this.networkCleanupCache.remove(neutronNetworkUuid);
1557             }
1558         }
1559         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1560     }
1561
1562     public void removeNetworkFromCleanupCache(String networkid) {
1563         NeutronNetwork network = null;
1564         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1565             if (neutronNetworkUuid.equals(networkid)) {
1566                 network = networkCleanupCache.get(neutronNetworkUuid);
1567                 break;
1568             }
1569         }
1570         if (network != null) {
1571             for (String neutronPortUuid:this.portCleanupCache.keySet()) {
1572                 if (this.portCleanupCache.get(neutronPortUuid).getNetworkUUID().equals(network.getNetworkUUID())) {
1573                     LOG.info("This network is used by another port", network);
1574                     return;
1575                 }
1576             }
1577             this.networkCleanupCache.remove(network.getNetworkUUID());
1578         }
1579     }
1580
1581     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1582         return this.networkCleanupCache;
1583     }
1584
1585     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1586         for (String neutronNetworkUuid:this.networkCleanupCache.keySet()) {
1587             if (neutronNetworkUuid.equals(networkid)) {
1588                 LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", networkid);
1589                 return networkCleanupCache.get(neutronNetworkUuid);
1590             }
1591         }
1592         return null;
1593     }
1594     /**
1595      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1596      *
1597      * @param ofPort the OF port number
1598      * @return the string with encoded OF port (example format "OFPort|999")
1599      */
1600     public static String encodeExcplicitOFPort(Long ofPort) {
1601         return "OFPort|" + ofPort.toString();
1602     }
1603     private void initNetworkCleanUpCache() {
1604         if (this.neutronNetworkCache != null) {
1605             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1606                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1607             }
1608         }
1609     }
1610     private void initPortCleanUpCache() {
1611         if (this.neutronPortCache != null) {
1612             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1613                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1614             }
1615         }
1616     }
1617     @Override
1618     public void setDependencies(ServiceReference serviceReference) {
1619         eventDispatcher =
1620                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1621         eventDispatcher.eventHandlerAdded(serviceReference, this);
1622         tenantNetworkManager =
1623                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1624         configurationService =
1625                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1626         arpProvider =
1627                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1628         inboundNatProvider =
1629                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1630         outboundNatProvider =
1631                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1632         routingProvider =
1633                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1634         l3ForwardingProvider =
1635                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1636         distributedArpService =
1637                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1638         nodeCacheManager =
1639                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1640         southbound =
1641                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1642         gatewayMacResolver =
1643                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1644         securityServicesManager =
1645                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1646
1647         initL3AdapterMembers();
1648     }
1649
1650     @Override
1651     public void setDependencies(Object impl) {
1652         if (impl instanceof INeutronNetworkCRUD) {
1653             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1654             initNetworkCleanUpCache();
1655         } else if (impl instanceof INeutronPortCRUD) {
1656             neutronPortCache = (INeutronPortCRUD)impl;
1657             initPortCleanUpCache();
1658         } else if (impl instanceof INeutronSubnetCRUD) {
1659             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1660         } else if (impl instanceof INeutronFloatingIPCRUD) {
1661             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1662         } else if (impl instanceof ArpProvider) {
1663             arpProvider = (ArpProvider)impl;
1664         } else if (impl instanceof InboundNatProvider) {
1665             inboundNatProvider = (InboundNatProvider)impl;
1666         } else if (impl instanceof OutboundNatProvider) {
1667             outboundNatProvider = (OutboundNatProvider)impl;
1668         } else if (impl instanceof RoutingProvider) {
1669             routingProvider = (RoutingProvider)impl;
1670         } else if (impl instanceof L3ForwardingProvider) {
1671             l3ForwardingProvider = (L3ForwardingProvider)impl;
1672         }else if (impl instanceof GatewayMacResolver) {
1673             gatewayMacResolver = (GatewayMacResolver)impl;
1674         }else if (impl instanceof IcmpEchoProvider) {
1675             icmpEchoProvider = (IcmpEchoProvider)impl;
1676         }
1677
1678         populateL3ForwardingCaches();
1679     }
1680 }