openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014, 2015 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import java.net.InetAddress;
  12 import java.net.UnknownHostException;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.concurrent.ExecutorService;
  20 import java.util.concurrent.Executors;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  25 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  26 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  27 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  40 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  41 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  42 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  43 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  44 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  49 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  50 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  51 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  52 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  53 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  54 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  55 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  56 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  57 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  58 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  59 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  69 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  70 import org.osgi.framework.ServiceReference;
  71 import org.slf4j.Logger;
  72 import org.slf4j.LoggerFactory;
  73
  74 import com.google.common.base.Preconditions;
  75 import com.google.common.util.concurrent.FutureCallback;
  76 import com.google.common.util.concurrent.Futures;
  77 import com.google.common.util.concurrent.ListenableFuture;
  78
  79 /**
  80  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  81  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  82  * as well as the multi-tenant router forwarding provider.
  83  */
  84 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  85     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  86
  87     // The implementation for each of these services is resolved by the OSGi Service Manager
  88     private volatile ConfigurationService configurationService;
  89     private volatile TenantNetworkManager tenantNetworkManager;
  90     private volatile NodeCacheManager nodeCacheManager;
  91     private volatile INeutronNetworkCRUD neutronNetworkCache;
  92     private volatile INeutronSubnetCRUD neutronSubnetCache;
  93     private volatile INeutronPortCRUD neutronPortCache;
  94     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  95     private volatile L3ForwardingProvider l3ForwardingProvider;
  96     private volatile InboundNatProvider inboundNatProvider;
  97     private volatile OutboundNatProvider outboundNatProvider;
  98     private volatile ArpProvider arpProvider;
  99     private volatile RoutingProvider routingProvider;
 100     private volatile GatewayMacResolver gatewayMacResolver;
 101     private volatile SecurityServicesManager securityServicesManager;
 102
 103     private class FloatIpData {
 104         // br-int of node where floating ip is associated with tenant port
 105         private final Long dpid;
 106         // patch port in br-int used to reach br-ex
 107         private final Long ofPort;
 108         // segmentation id of the net where fixed ip is instantiated
 109         private final String segId;
 110         // mac address assigned to neutron port of floating ip
 111         private final String macAddress;
 112         private final String floatingIpAddress;
 113         // ip address given to tenant vm
 114         private final String fixedIpAddress;
 115         private final String neutronRouterMac;
 116
 117         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 118                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 119             this.dpid = dpid;
 120             this.ofPort = ofPort;
 121             this.segId = segId;
 122             this.macAddress = macAddress;
 123             this.floatingIpAddress = floatingIpAddress;
 124             this.fixedIpAddress = fixedIpAddress;
 125             this.neutronRouterMac = neutronRouterMac;
 126         }
 127     }
 128
 129     private Map<String, String> networkIdToRouterMacCache;
 130     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 131     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 132
 133     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 134     private Map<String, FloatIpData> floatIpDataMapCache;
 135
 136     private String externalRouterMac;
 137     private Boolean enabled = false;
 138     private Boolean flgDistributedARPEnabled = true;
 139     private Boolean isCachePopulationDone = false;
 140     private final ExecutorService gatewayMacResolverPool = Executors.newFixedThreadPool(5);
 141
 142     private Southbound southbound;
 143     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 144
 145     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 146     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 147     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 148     private static final String OWNER_FLOATING_IP = "network:floatingip";
 149     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 150
 151     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 152         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 153         this.neutronModelsDataStoreHelper = neutronHelper;
 154     }
 155
 156     private void initL3AdapterMembers() {
 157         Preconditions.checkNotNull(configurationService);
 158
 159         if (configurationService.isL3ForwardingEnabled()) {
 160             this.networkIdToRouterMacCache = new HashMap<>();
 161             this.networkIdToRouterIpListCache = new HashMap<>();
 162             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 163             this.neutronPortToDpIdCache = new HashMap<>();
 164             this.floatIpDataMapCache = new HashMap<>();
 165
 166             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 167             if (this.externalRouterMac == null) {
 168                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 169             }
 170
 171             this.enabled = true;
 172             LOG.info("OVSDB L3 forwarding is enabled");
 173             if (configurationService.isDistributedArpDisabled()) {
 174                 this.flgDistributedARPEnabled = false;
 175                 LOG.info("Distributed ARP responder is disabled");
 176             } else {
 177                 LOG.debug("Distributed ARP responder is enabled");
 178             }
 179         } else {
 180             LOG.debug("OVSDB L3 forwarding is disabled");
 181         }
 182     }
 183
 184     //
 185     // Callbacks from AbstractHandler
 186     //
 187     @Override
 188     public void processEvent(AbstractEvent abstractEvent) {
 189         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 190             LOG.error("Unable to process abstract event " + abstractEvent);
 191             return;
 192         }
 193         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 194         switch (ev.getAction()) {
 195             case UPDATE:
 196                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 197                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 198                 } else {
 199                     LOG.warn("Received update for an unexpected event " + ev);
 200                 }
 201                 break;
 202             case ADD:
 203                 // fall through...
 204                 // break;
 205             case DELETE:
 206                 // fall through...
 207                 // break;
 208             default:
 209                 LOG.warn("Unable to process event " + ev);
 210                 break;
 211         }
 212     }
 213
 214     //
 215     // Callbacks from GatewayMacResolverListener
 216     //
 217
 218     @Override
 219     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 220         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 221                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 222         if (!this.enabled) {
 223             return;
 224         }
 225
 226         if (macAddress == null || macAddress.getValue() == null) {
 227             // TODO: handle cases when mac is null
 228             return;
 229         }
 230
 231         //
 232         // Enqueue event so update is handled by adapter's thread
 233         //
 234         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 235     }
 236
 237     private void populateL3ForwardingCaches() {
 238         if (!this.enabled) {
 239             return;
 240         }
 241         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 242                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 243             return;
 244         }
 245         this.isCachePopulationDone = true;
 246         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 247         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 248         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 249         if(routers != null && routers.getRouter() != null && ports != null) {
 250             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 251             for( Router router : routers.getRouter()) {
 252                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 253                 if(!ports.getPort().isEmpty()) {
 254                     for( Port port : ports.getPort()) {
 255                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 256                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 257                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 258                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 259                                     , port.getMacAddress());
 260
 261                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 262                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 263                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 264                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 265                         }
 266                     }
 267                 }else {
 268                     LOG.warn("L3 Cache Population :Did not find any port information " +
 269                             "in config Data Store for router {}",router);
 270                 }
 271             }
 272         }
 273         LOG.debug("NetVirt L3 caches population is done");
 274     }
 275
 276     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 277         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 278             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 279             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 280             for(Node bridge : bridges) {
 281                 List<OvsdbTerminationPointAugmentation> interfaces
 282                         = southbound.extractTerminationPointAugmentations(bridge);
 283                 if(interfaces != null && !interfaces.isEmpty()) {
 284                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 285                             interfaces.size(), bridge.getNodeId());
 286                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 287                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 288                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 289                             Long dpId = getDpidForIntegrationBridge(bridge);
 290                             Uuid interfaceUuid = intf.getInterfaceUuid();
 291                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 292                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 293                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 294                             break;
 295                         }
 296                     }
 297                 }
 298             }
 299         }
 300         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 301     }
 302
 303     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 304         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 305         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 306         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 307             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 308                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 309                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 310                             "Fetching from data store.",neutronFloatingIP.getID());
 311                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 312                 }
 313             }
 314         }
 315         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 316         return floatIpDataMapCache.values();
 317     }
 318     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 319         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 320                 "floating id {} ",neutronFloatingId);
 321         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 322             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 323             if (neutronFloatingIP == null) {
 324                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 325                 return null;
 326             }
 327             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 328             NeutronPort neutronPortForFloatIp = null;
 329             for (NeutronPort neutronPort : neutronPorts) {
 330                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 331                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 332                     neutronPortForFloatIp = neutronPort;
 333                     break;
 334                 }
 335             }
 336
 337             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 338             if(neutronTenantPortUuid == null) {
 339                 return null;
 340             }
 341             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 342             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 343             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 344             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 345
 346             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 347             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 348                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 349             String providerSegmentationId = tenantNeutronNetwork != null ?
 350                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 351             String neutronRouterMac = tenantNeutronNetwork != null ?
 352                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 353
 354             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 355                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 356                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 357                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 358                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 359                                 "seg {} mac {} rtrMac {}",
 360                         fixedIpAddress,
 361                         floatingIpAddress,
 362                         neutronPortForFloatIp,
 363                         neutronTenantPortUuid,
 364                         providerSegmentationId,
 365                         floatingIpMac,
 366                         neutronRouterMac);
 367
 368                 return null;
 369             }
 370
 371             // get ofport for patch port in br-int
 372             final Long dpId = nodeIfPair.getLeft();
 373             final Long ofPort = findOFPortForExtPatch(dpId);
 374             if (ofPort == null) {
 375                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 376                                 "to connect floating ip to external bridge. dpid {}",
 377                         dpId);
 378                 return null;
 379             }
 380
 381             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 382                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 383             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 384
 385         }
 386         return floatIpDataMapCache.get(neutronFloatingId);
 387     }
 388     /**
 389      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 390      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 391      * subscribe so that the mac address associated to the gateway ip address is available.
 392      *
 393      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 394      */
 395     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 396         Preconditions.checkNotNull(externalRouterMacUpdate);
 397
 398         flushExistingIpRewrite();
 399         this.externalRouterMac = externalRouterMacUpdate;
 400         rebuildExistingIpRewrite();
 401     }
 402
 403     /**
 404      * Process the event.
 405      *
 406      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 407      * @param subnet An instance of NeutronSubnet object.
 408      */
 409     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 410         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 411     }
 412
 413     /**
 414      * Process the port event as a router interface event.
 415      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 416      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 417      *
 418      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 419      * @param neutronPort An instance of NeutronPort object.
 420      */
 421     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 422         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 423
 424         this.processSecurityGroupUpdate(neutronPort);
 425         if (!this.enabled) {
 426             return;
 427         }
 428
 429         final boolean isDelete = action == Action.DELETE;
 430
 431         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 432             if(!isDelete){
 433                 Node externalBridgeNode = getExternalBridgeNode();
 434                 if(externalBridgeNode != null){
 435                     LOG.info("Port {} is network router gateway interface, "
 436                             + "triggering gateway resolution for the attached external network on node {}", neutronPort, externalBridgeNode);
 437                     this.triggerGatewayMacResolver(externalBridgeNode, neutronPort);
 438                 }else{
 439                     LOG.error("Did not find Node that has external bridge (br-ex), Gateway resolution failed");
 440                 }
 441             }else{
 442                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 443
 444                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 445                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 446                     // TODO support IPv6
 447                     if (externalSubnet != null &&
 448                             externalSubnet.getIpVersion() == 4) {
 449                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 450                     }
 451                 }
 452             }
 453         }
 454
 455         // Treat the port event as a router interface event if the port belongs to router. This is a
 456         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 457         //
 458         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 459             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 460
 461             if (neutronPort.getFixedIPs() != null) {
 462                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 463                     NeutronRouter_Interface neutronRouterInterface =
 464                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 465                     // id of router interface to be same as subnet
 466                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 467                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 468
 469                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 470                 }
 471             }
 472         } else {
 473             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 474             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 475             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 476             // there.
 477             //
 478             if (!isDelete && neutronPort.getFixedIPs() != null) {
 479                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 480                     NeutronRouter_Interface neutronRouterInterface =
 481                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 482                     if (neutronRouterInterface != null) {
 483                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 484                     }
 485                 }
 486             }
 487             this.updateL3ForNeutronPort(neutronPort, isDelete);
 488         }
 489     }
 490
 491     /**
 492      * Process the event.
 493      *
 494      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 495      * @param neutronRouter An instance of NeutronRouter object.
 496      */
 497     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 498         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 499     }
 500
 501     /**
 502      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 503      * delete the ports on the same subnet.
 504      *
 505      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 506      * @param neutronRouter An instance of NeutronRouter object.
 507      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 508      */
 509     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 510                                                   final NeutronRouter_Interface neutronRouterInterface,
 511                                                   Action action) {
 512         LOG.debug("Router interface {} got event {}. Subnet {}",
 513                      neutronRouterInterface.getPortUUID(),
 514                      action,
 515                      neutronRouterInterface.getSubnetUUID());
 516         if (!this.enabled) {
 517             return;
 518         }
 519
 520         final boolean isDelete = action == Action.DELETE;
 521
 522         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 523
 524         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 525         // see if they are affected by l3
 526         //
 527         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 528             boolean currPortShouldBeDeleted = false;
 529             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 530             if (isDelete) {
 531                 if (neutronPort.getFixedIPs() != null) {
 532                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 533                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 534                             currPortShouldBeDeleted = true;
 535                             break;
 536                         }
 537                     }
 538                 }
 539             }
 540             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 541         }
 542
 543         if (isDelete) {
 544             /*
 545              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 546              */
 547             this.cleanupRouterCache(neutronRouterInterface);
 548         }
 549     }
 550
 551     /**
 552      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 553      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 554      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 555      * way around) on OpenFlow Table 100.
 556      *
 557      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 558      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 559      */
 560     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 561                                              Action actionIn) {
 562         Preconditions.checkNotNull(neutronFloatingIP);
 563
 564         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 565                 neutronFloatingIP.getFixedIPAddress(),
 566                 neutronFloatingIP.getFloatingIPAddress(),
 567                 neutronFloatingIP.getFloatingNetworkUUID());
 568         if (!this.enabled) {
 569             return;
 570         }
 571
 572         Action action;
 573
 574         // Consider action to be delete if getFixedIPAddress is null
 575         //
 576         if (neutronFloatingIP.getFixedIPAddress() == null) {
 577             action = Action.DELETE;
 578         } else {
 579             action = actionIn;
 580         }
 581
 582         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 583
 584         if (action != Action.DELETE) {
 585             // must be first, as it updates floatIpDataMapCache
 586             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 587
 588             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 589             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 590         } else {
 591             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 592             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 593
 594             // must be last, as it updates floatIpDataMapCache
 595             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 596         }
 597     }
 598
 599     /**
 600      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 601      * ip, otherwise for newer one.
 602      */
 603     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 604         Preconditions.checkNotNull(neutronFloatingIP);
 605
 606         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 607         if (fid == null) {
 608             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 609                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 610             return;
 611         }
 612         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 613                                       action);
 614     }
 615
 616     /**
 617      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 618      * ip, otherwise for newer one.
 619      */
 620     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 621         Preconditions.checkNotNull(neutronFloatingIP);
 622
 623         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 624         if (fid == null) {
 625             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 626                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 627             return;
 628         }
 629         programOutboundIpRewriteStage1(fid, action);
 630     }
 631
 632     private void flushExistingIpRewrite() {
 633         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 634             programOutboundIpRewriteStage1(fid, Action.DELETE);
 635         }
 636     }
 637
 638     private void rebuildExistingIpRewrite() {
 639         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 640             programOutboundIpRewriteStage1(fid, Action.ADD);
 641         }
 642     }
 643
 644     /**
 645      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 646      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 647      * requests.
 648      */
 649     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 650         Preconditions.checkNotNull(neutronFloatingIP);
 651         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 652         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 653
 654         // find bridge Node where floating ip is configured by looking up cache for its port
 655         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 656         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 657         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 658         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 659         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 660         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 661
 662         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 663         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 664                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 665         final String providerSegmentationId = tenantNeutronNetwork != null ?
 666                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 667         final String neutronRouterMac = tenantNeutronNetwork != null ?
 668                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 669
 670         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 671                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 672                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 673                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 674             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 675                     fixedIpAddress,
 676                     floatingIpAddress,
 677                     neutronPortForFloatIp,
 678                     neutronTenantPortUuid,
 679                     providerSegmentationId,
 680                     floatingIpMac,
 681                     neutronRouterMac);
 682             return;
 683         }
 684
 685         // get ofport for patch port in br-int
 686         final Long dpId = nodeIfPair.getLeft();
 687         final Long ofPort = findOFPortForExtPatch(dpId);
 688         if (ofPort == null) {
 689             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 690                     dpId);
 691             return;
 692         }
 693
 694         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 695         //
 696         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 697                 Action.ADD)) {
 698             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 699                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 700             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 701             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 702                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 703                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 704         }
 705     }
 706
 707     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 708         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 709         if (floatIpData == null) {
 710             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 711             return;
 712         }
 713
 714         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 715                 floatIpData.floatingIpAddress, Action.DELETE)) {
 716             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 717             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 718                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 719         }
 720     }
 721
 722     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 723         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 724             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 725                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 726                 return neutronPort;
 727             }
 728         }
 729         return null;
 730     }
 731
 732     private Long findOFPortForExtPatch(Long dpId) {
 733         final String brInt = configurationService.getIntegrationBridgeName();
 734         final String brExt = configurationService.getExternalBridgeName();
 735         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 736
 737         Preconditions.checkNotNull(dpId);
 738         Preconditions.checkNotNull(portNameInt);
 739
 740         final long dpidPrimitive = dpId;
 741         for (Node node : nodeCacheManager.getBridgeNodes()) {
 742             if (dpidPrimitive == southbound.getDataPathId(node)) {
 743                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 744                         southbound.getTerminationPointOfBridge(node, portNameInt);
 745                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 746             }
 747         }
 748         return null;
 749     }
 750
 751     /**
 752      * Process the event.
 753      *
 754      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 755      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 756      */
 757     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 758         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 759     }
 760
 761     //
 762     // Callbacks from OVSDB's southbound handler
 763     //
 764     /**
 765      * Process the event.
 766      *
 767      * @param bridgeNode An instance of Node object.
 768      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 769      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 770      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 771      * object.
 772      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 773      */
 774     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 775                                      final NeutronNetwork neutronNetwork, Action action) {
 776         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 777                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 778         if (!this.enabled) {
 779             return;
 780         }
 781
 782         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 783         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 784         final Uuid interfaceUuid = intf.getInterfaceUuid();
 785
 786         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 787                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 788
 789         if (neutronPort != null) {
 790             final String neutronPortUuid = neutronPort.getPortUUID();
 791
 792             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 793                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 794             }
 795
 796             handleNeutronPortEvent(neutronPort, action);
 797         }
 798
 799         if (action == Action.DELETE && interfaceUuid != null) {
 800             handleInterfaceEventDelete(intf, dpId);
 801         }
 802     }
 803
 804     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 805         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 806         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 807                 neutronPortUuid, dpId, interfaceUuid.getValue());
 808     }
 809
 810     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 811         // Remove entry from neutronPortToDpIdCache based on interface uuid
 812         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 813             final String currPortUuid = entry.getKey();
 814             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 815                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 816                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 817                 neutronPortToDpIdCache.remove(currPortUuid);
 818                 break;
 819             }
 820         }
 821     }
 822
 823     //
 824     // Internal helpers
 825     //
 826     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 827
 828         final String networkUUID = neutronPort.getNetworkUUID();
 829         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 830
 831         // If there is no router interface handling the networkUUID, we are done
 832         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 833             return;
 834         }
 835
 836         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 837         // router interface are handled via handleNeutronRouterInterfaceEvent.
 838         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 839             return;
 840         }
 841
 842         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 843         final String providerSegmentationId = neutronNetwork != null ?
 844                                               neutronNetwork.getProviderSegmentationID() : null;
 845         final String tenantMac = neutronPort.getMacAddress();
 846
 847         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 848             tenantMac == null || tenantMac.isEmpty()) {
 849             // done: go no further w/out all the info needed...
 850             return;
 851         }
 852
 853         final Action action = isDelete ? Action.DELETE : Action.ADD;
 854         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 855         if (nodes.isEmpty()) {
 856             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 857         }
 858         for (Node node : nodes) {
 859             final Long dpid = getDpidForIntegrationBridge(node);
 860             if (dpid == null) {
 861                 continue;
 862             }
 863             if (neutronPort.getFixedIPs() == null) {
 864                 continue;
 865             }
 866             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 867                 final String tenantIpStr = neutronIP.getIpAddress();
 868                 if (tenantIpStr.isEmpty()) {
 869                     continue;
 870                 }
 871
 872                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 873                 // still needed when routing to subnets non-local to node (bug 2076).
 874                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 875
 876                 // Configure distributed ARP responder
 877                 if (flgDistributedARPEnabled) {
 878                     // Arp rule is only needed when segmentation exists in the given node (bug 4752).
 879                     boolean arpNeeded = tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 880                     final Action actionForNode = arpNeeded ? action : Action.DELETE;
 881                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 882                 }
 883             }
 884         }
 885     }
 886
 887     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 888         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 889         /**
 890          * Get updated data and original data for the the changed. Identify the security groups that got
 891          * added and removed and call the appropriate providers for updating the flows.
 892          */
 893         try {
 894             NeutronPort originalPort = neutronPort.getOriginalPort();
 895             if (null == originalPort) {
 896                 LOG.debug("processSecurityGroupUpdate: originalport is empty");
 897                 return;
 898             }
 899             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 900                                                                             neutronPort.getOriginalPort());
 901             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 902                                                                               neutronPort);
 903
 904             if (null != addedGroup && !addedGroup.isEmpty()) {
 905                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 906             }
 907             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 908                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 909             }
 910
 911         } catch (Exception e) {
 912             LOG.error("Exception in processSecurityGroupUpdate", e);
 913         }
 914     }
 915
 916     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 917         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 918         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 919         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 920         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 921             NeutronSecurityGroup securityGroup1 = iterator.next();
 922             for (NeutronSecurityGroup securityGroup2 :list2) {
 923                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 924                     iterator.remove();
 925                 }
 926             }
 927         }
 928         return list1;
 929     }
 930
 931     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 932                                            String macAddress, String ipStr,
 933                                            Action actionForNode) {
 934         if (actionForNode == Action.DELETE) {
 935             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 936                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 937         }
 938         if (actionForNode == Action.ADD) {
 939             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 940                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 941         }
 942
 943         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 944                                                        macAddress, ipStr, actionForNode);
 945     }
 946
 947     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 948                                              String macAddress,
 949                                              String address,
 950                                              Action actionForNode) {
 951         Status status;
 952         try {
 953             InetAddress inetAddress = InetAddress.getByName(address);
 954             status = l3ForwardingProvider == null ?
 955                      new Status(StatusCode.SUCCESS) :
 956                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 957                                                                       inetAddress, macAddress, actionForNode);
 958         } catch (UnknownHostException e) {
 959             status = new Status(StatusCode.BADREQUEST);
 960         }
 961
 962         if (status.isSuccess()) {
 963             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 964                          l3ForwardingProvider == null ? "skipped" : "programmed",
 965                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 966         } else {
 967             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 968                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 969         }
 970         return status;
 971     }
 972
 973     // --
 974
 975     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 976                                                        Boolean isDelete) {
 977         Preconditions.checkNotNull(destNeutronRouterInterface);
 978
 979         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 980         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 981         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 982         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 983         final NeutronNetwork neutronNetwork = subnet != null ?
 984                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 985         final String destinationSegmentationId = neutronNetwork != null ?
 986                                                  neutronNetwork.getProviderSegmentationID() : null;
 987         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 988         final String cidr = subnet != null ? subnet.getCidr() : null;
 989         final int mask = getMaskLenFromCidr(cidr);
 990
 991         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 992                      destNeutronRouterInterface, isDelete);
 993
 994         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 995         // the local cache
 996         if (neutronNetwork != null) {
 997             if (macAddress == null || macAddress.isEmpty()) {
 998                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 999             }
1000             if (ipList == null || ipList.isEmpty()) {
1001                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1002             }
1003         }
1004
1005         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1006             cidr == null || cidr.isEmpty() ||
1007             macAddress == null || macAddress.isEmpty() ||
1008             ipList == null || ipList.isEmpty()) {
1009             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1010                          destinationSegmentationId, cidr, macAddress, ipList);
1011             // done: go no further w/out all the info needed...
1012             return;
1013         }
1014
1015         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1016
1017         // Keep cache for finding router's mac from network uuid -- add
1018         //
1019         if (! isDelete) {
1020             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1021             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1022             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1023         }
1024
1025         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1026         if (nodes.isEmpty()) {
1027             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1028         }
1029         for (Node node : nodes) {
1030             final Long dpid = getDpidForIntegrationBridge(node);
1031             if (dpid == null) {
1032                 continue;
1033             }
1034
1035             for (Neutron_IPs neutronIP : ipList) {
1036                 final String ipStr = neutronIP.getIpAddress();
1037                 if (ipStr.isEmpty()) {
1038                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1039                             node.getNodeId().getValue(), ipStr);
1040                     continue;
1041                 }
1042
1043                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1044                 //
1045                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1046                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1047                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1048                                                               neutronNetwork, destinationSegmentationId,
1049                                                               macAddress, ipStr, mask, actionForNode,
1050                                                               true /*isReflexsive*/);
1051                 }
1052
1053                 if (! isExternal) {
1054                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1055                             actionForNode);
1056                 }
1057                 // Enable ARP responder by default, because router interface needs to be responded always.
1058                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1059             }
1060
1061             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1062             // for the external neutron networks.
1063             //
1064             {
1065                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1066                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1067             }
1068         }
1069
1070         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1071     }
1072
1073     private void programFlowForNetworkFromExternal(final Node node,
1074                                                    final Long dpid,
1075                                                    final String destinationSegmentationId,
1076                                                    final String dstMacAddress,
1077                                                    final String destIpStr,
1078                                                    final int destMask,
1079                                                    final Action actionForNode) {
1080         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1081                 dstMacAddress, destIpStr, destMask, actionForNode);
1082     }
1083
1084     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1085                                                            final Long dpid,
1086                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1087                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1088                                                            final NeutronNetwork dstNeutronNetwork,
1089                                                            final String destinationSegmentationId,
1090                                                            final String dstMacAddress,
1091                                                            final String destIpStr,
1092                                                            final int destMask,
1093                                                            final Action actionForNode,
1094                                                            Boolean isReflexsive) {
1095         Preconditions.checkNotNull(srcNeutronRouterInterface);
1096         Preconditions.checkNotNull(dstNeutronRouterInterface);
1097
1098         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1099         if (sourceSubnetId == null) {
1100             LOG.error("Could not get provider Subnet ID from router interface {}",
1101                          srcNeutronRouterInterface.getID());
1102             return;
1103         }
1104
1105         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1106         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1107         if (sourceNetworkId == null) {
1108             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1109             return;
1110         }
1111
1112         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1113         if (sourceNetwork == null) {
1114             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1115             return;
1116         }
1117
1118         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1119             // Isolate subnets from different tenants within the same router
1120             return;
1121         }
1122         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1123         if (sourceSegmentationId == null) {
1124             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1125             return;
1126         }
1127         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1128             // Skip 'self'
1129             return;
1130         }
1131
1132         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1133                                      dstMacAddress, destIpStr, destMask, actionForNode);
1134
1135         // Flip roles src->dst; dst->src
1136         if (isReflexsive) {
1137             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1138             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1139             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1140             final String cidr2 = sourceSubnet.getCidr();
1141             final int mask2 = getMaskLenFromCidr(cidr2);
1142
1143             if (cidr2 == null || cidr2.isEmpty() ||
1144                 macAddress2 == null || macAddress2.isEmpty() ||
1145                 ipList2 == null || ipList2.isEmpty()) {
1146                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1147                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1148                 // done: go no further w/out all the info needed...
1149                 return;
1150             }
1151
1152             for (Neutron_IPs neutronIP2 : ipList2) {
1153                 final String ipStr2 = neutronIP2.getIpAddress();
1154                 if (ipStr2.isEmpty()) {
1155                     continue;
1156                 }
1157                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1158                                                           srcNeutronRouterInterface,
1159                                                           sourceNetwork, sourceSegmentationId,
1160                                                           macAddress2, ipStr2, mask2, actionForNode,
1161                                                           false /*isReflexsive*/);
1162             }
1163         }
1164     }
1165
1166     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1167                                               String destinationSegmentationId,
1168                                               String macAddress, String ipStr, int mask,
1169                                               Action actionForNode) {
1170         if (actionForNode == Action.DELETE) {
1171             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1172                          " action {}",
1173                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1174                          macAddress, ipStr, mask, actionForNode);
1175         }
1176         if (actionForNode == Action.ADD) {
1177             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1178                          " action {}",
1179                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1180                          macAddress, ipStr, mask, actionForNode);
1181         }
1182
1183         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1184                                                           macAddress, ipStr, mask, actionForNode);
1185     }
1186
1187     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1188                                                 String destinationSegmentationId,
1189                                                 String macAddress,
1190                                                 String address, int mask,
1191                                                 Action actionForNode) {
1192         Status status;
1193         try {
1194             InetAddress inetAddress = InetAddress.getByName(address);
1195             status = routingProvider == null ?
1196                      new Status(StatusCode.SUCCESS) :
1197                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1198                                                             macAddress, inetAddress, mask, actionForNode);
1199         } catch (UnknownHostException e) {
1200             status = new Status(StatusCode.BADREQUEST);
1201         }
1202
1203         if (status.isSuccess()) {
1204             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1205                          routingProvider == null ? "skipped" : "programmed",
1206                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1207                          actionForNode);
1208         } else {
1209             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1210                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1211                          actionForNode, status);
1212         }
1213         return status;
1214     }
1215
1216     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1217                                            String macAddress, String ipStr,
1218                                            Action action) {
1219         if (action == Action.DELETE ) {
1220             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1221                     dpid, segOrOfPort, macAddress, ipStr, action);
1222         }
1223         if (action == Action.ADD) {
1224             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
1225                     dpid, segOrOfPort, macAddress, ipStr, action);
1226         }
1227
1228         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1229         return status.isSuccess();
1230     }
1231
1232     private Status programStaticArpStage2(Long dpid,
1233                                           String segOrOfPort,
1234                                           String macAddress,
1235                                           String address,
1236                                           Action action) {
1237         Status status;
1238         try {
1239             InetAddress inetAddress = InetAddress.getByName(address);
1240             status = arpProvider == null ?
1241                      new Status(StatusCode.SUCCESS) :
1242                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1243                                                        macAddress, inetAddress, action);
1244         } catch (UnknownHostException e) {
1245             status = new Status(StatusCode.BADREQUEST);
1246         }
1247
1248         if (status.isSuccess()) {
1249             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1250                          arpProvider == null ? "skipped" : "programmed",
1251                          macAddress, address, dpid, segOrOfPort, action);
1252         } else {
1253             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1254                          macAddress, address, dpid, segOrOfPort, action, status);
1255         }
1256         return status;
1257     }
1258
1259     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1260                                                   String matchAddress, String rewriteAddress,
1261                                                   Action action) {
1262         if (action == Action.DELETE ) {
1263             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1264                     " action {}",
1265                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1266         }
1267         if (action == Action.ADD ) {
1268             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1269                     " action {}",
1270                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1271         }
1272
1273         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1274                 rewriteAddress, action);
1275         return status.isSuccess();
1276     }
1277
1278     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1279                                                  String matchAddress, String rewriteAddress,
1280                                                  Action action) {
1281         Status status;
1282         try {
1283             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1284             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1285             status = inboundNatProvider == null ?
1286                     new Status(StatusCode.SUCCESS) :
1287                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1288                             inetMatchAddress, inetRewriteAddress,
1289                             action);
1290         } catch (UnknownHostException e) {
1291             status = new Status(StatusCode.BADREQUEST);
1292         }
1293
1294         if (status.isSuccess()) {
1295             final boolean isSkipped = inboundNatProvider == null;
1296             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1297                     isSkipped ? "skipped" : "programmed",
1298                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1299         } else {
1300             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1301                          " status:{}",
1302                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1303                     status);
1304         }
1305         return status;
1306     }
1307
1308     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1309                                                  Action actionForRewriteExclusion) {
1310         if (actionForRewriteExclusion == Action.DELETE ) {
1311             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1312                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1313         }
1314         if (actionForRewriteExclusion == Action.ADD) {
1315             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1316                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1317         }
1318
1319         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1320     }
1321
1322     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1323                                                    Action actionForNode) {
1324         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1325                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1326
1327         if (status.isSuccess()) {
1328             final boolean isSkipped = outboundNatProvider == null;
1329             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1330                          isSkipped ? "skipped" : "programmed",
1331                          cidr, node.getNodeId().getValue(), actionForNode);
1332         } else {
1333             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1334                          cidr, node.getNodeId().getValue(), actionForNode, status);
1335         }
1336         return status;
1337     }
1338
1339     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1340
1341         if (action == Action.DELETE) {
1342             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1343                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1344         }
1345         if (action == Action.ADD) {
1346             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1347                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1348         }
1349
1350         this.programOutboundIpRewriteStage2(fid, action);
1351     }
1352
1353     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1354         Status status;
1355         try {
1356             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1357             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1358             status = outboundNatProvider == null ?
1359                     new Status(StatusCode.SUCCESS) :
1360                     outboundNatProvider.programIpRewriteRule(
1361                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1362                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1363         } catch (UnknownHostException e) {
1364             status = new Status(StatusCode.BADREQUEST);
1365         }
1366
1367         if (status.isSuccess()) {
1368             final boolean isSkipped = outboundNatProvider == null;
1369             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1370                             " action {}",
1371                          isSkipped ? "skipped" : "programmed",
1372                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1373         } else {
1374             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1375                          " action {} status:{}",
1376                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1377         }
1378         return status;
1379     }
1380
1381     private int getMaskLenFromCidr(String cidr) {
1382         if (cidr == null) {
1383             return 0;
1384         }
1385         String[] splits = cidr.split("/");
1386         if (splits.length != 2) {
1387             return 0;
1388         }
1389
1390         int result;
1391         try {
1392             result = Integer.parseInt(splits[1].trim());
1393         } catch (NumberFormatException nfe) {
1394             result = 0;
1395         }
1396         return result;
1397     }
1398
1399     private Long getDpidForIntegrationBridge(Node node) {
1400         // Check if node is integration bridge; and only then return its dpid
1401         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1402             return southbound.getDataPathId(node);
1403         }
1404         return null;
1405     }
1406
1407     private Long getDpidForExternalBridge(Node node) {
1408         // Check if node is integration bridge; and only then return its dpid
1409         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1410             return southbound.getDataPathId(node);
1411         }
1412         return null;
1413     }
1414
1415     private Node getExternalBridgeNode(){
1416         //Pickup the first node that has external bridge (br-ex).
1417         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1418         //the external network is reachable from every br-ex
1419         // TODO: Consider other deployment scenario, and thing of better solution.
1420         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1421         for(Node node : allBridges){
1422             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1423                 return node;
1424             }
1425         }
1426         return null;
1427     }
1428
1429     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1430         if (gatewayPort.getFixedIPs() == null) {
1431             return null;
1432         }
1433         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1434             String subnetUUID = neutronIPs.getSubnetUUID();
1435             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1436             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1437                 return extSubnet;
1438             }
1439             if (extSubnet == null) {
1440                 // TODO: when subnet is created, try again.
1441                 LOG.debug("subnet {} in not found", subnetUUID);
1442              }
1443         }
1444         return null;
1445     }
1446
1447      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1448          /*
1449           *  Fix for 4277
1450           *  Remove the router cache only after deleting the neutron
1451           *  port l3 flows.
1452           */
1453          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1454
1455          if (neutronPort != null) {
1456              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1457              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1458              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1459          }
1460      }
1461
1462     public void triggerGatewayMacResolver(final Node node, final NeutronPort gatewayPort ){
1463
1464         Preconditions.checkNotNull(node);
1465         Preconditions.checkNotNull(gatewayPort);
1466         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1467
1468         if(externalNetwork != null){
1469             if(externalNetwork.isRouterExternal()){
1470                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1471
1472                 // TODO: address IPv6 case.
1473                 if (externalSubnet != null &&
1474                     externalSubnet.getIpVersion() == 4 &&
1475                     gatewayPort.getFixedIPs() != null) {
1476                     LOG.info("Trigger MAC resolution for gateway ip {} on Node {}",externalSubnet.getGatewayIP(),node.getNodeId());
1477                     ListenableFuture<MacAddress> gatewayMacAddress =
1478                         gatewayMacResolver.resolveMacAddress(this,
1479                                                              getDpidForExternalBridge(node),
1480                                                              new Ipv4Address(externalSubnet.getGatewayIP()),
1481                                                              new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1482                                                              new MacAddress(gatewayPort.getMacAddress()),
1483                                                              true);
1484                     if(gatewayMacAddress != null){
1485                         Futures.addCallback(gatewayMacAddress, new FutureCallback<MacAddress>(){
1486                             @Override
1487                             public void onSuccess(MacAddress result) {
1488                                 if(result != null){
1489                                     if(!result.getValue().equals(externalRouterMac)){
1490                                         updateExternalRouterMac(result.getValue());
1491                                         LOG.info("Resolved MAC address for gateway IP {} is {}", externalSubnet.getGatewayIP(),result.getValue());
1492                                     }
1493                                 }else{
1494                                     LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1495                                 }
1496                             }
1497
1498                             @Override
1499                             public void onFailure(Throwable t) {
1500                                 LOG.warn("MAC address resolution failed for gateway IP {}", externalSubnet.getGatewayIP());
1501                             }
1502                         }, gatewayMacResolverPool);
1503                     }
1504                 } else {
1505                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1506                 }
1507             }
1508         }else{
1509             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1510         }
1511     }
1512
1513     /**
1514      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1515      *
1516      * @param ofPort the OF port number
1517      * @return the string with encoded OF port (example format "OFPort|999")
1518      */
1519     public static String encodeExcplicitOFPort(Long ofPort) {
1520         return "OFPort|" + ofPort.toString();
1521     }
1522
1523     @Override
1524     public void setDependencies(ServiceReference serviceReference) {
1525         eventDispatcher =
1526                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1527         eventDispatcher.eventHandlerAdded(serviceReference, this);
1528         tenantNetworkManager =
1529                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1530         configurationService =
1531                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1532         arpProvider =
1533                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1534         inboundNatProvider =
1535                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1536         outboundNatProvider =
1537                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1538         routingProvider =
1539                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1540         l3ForwardingProvider =
1541                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1542         nodeCacheManager =
1543                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1544         southbound =
1545                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1546         gatewayMacResolver =
1547                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1548         securityServicesManager =
1549                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1550         initL3AdapterMembers();
1551     }
1552
1553     @Override
1554     public void setDependencies(Object impl) {
1555         if (impl instanceof INeutronNetworkCRUD) {
1556             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1557         } else if (impl instanceof INeutronPortCRUD) {
1558             neutronPortCache = (INeutronPortCRUD)impl;
1559         } else if (impl instanceof INeutronSubnetCRUD) {
1560             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1561         } else if (impl instanceof INeutronFloatingIPCRUD) {
1562             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1563         } else if (impl instanceof ArpProvider) {
1564             arpProvider = (ArpProvider)impl;
1565         } else if (impl instanceof InboundNatProvider) {
1566             inboundNatProvider = (InboundNatProvider)impl;
1567         } else if (impl instanceof OutboundNatProvider) {
1568             outboundNatProvider = (OutboundNatProvider)impl;
1569         } else if (impl instanceof RoutingProvider) {
1570             routingProvider = (RoutingProvider)impl;
1571         } else if (impl instanceof L3ForwardingProvider) {
1572             l3ForwardingProvider = (L3ForwardingProvider)impl;
1573         }else if (impl instanceof GatewayMacResolver) {
1574             gatewayMacResolver = (GatewayMacResolver)impl;
1575         }
1576         populateL3ForwardingCaches();
1577     }
1578 }