openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  15 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  16 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  17 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  18 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  19 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.IcmpEchoProvider;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  37 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  38 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  39 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  49 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  50 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  60 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  61 import org.osgi.framework.ServiceReference;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 import java.net.InetAddress;
  66 import java.net.UnknownHostException;
  67 import java.util.ArrayList;
  68 import java.util.Collection;
  69 import java.util.HashMap;
  70 import java.util.HashSet;
  71 import java.util.Iterator;
  72 import java.util.List;
  73 import java.util.Map;
  74 import java.util.Set;
  75
  76 /**
  77  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  78  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  79  * as well as the multi-tenant router forwarding provider.
  80  */
  81 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  82     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  83
  84     // The implementation for each of these services is resolved by the OSGi Service Manager
  85     private volatile ConfigurationService configurationService;
  86     private volatile TenantNetworkManager tenantNetworkManager;
  87     private volatile NodeCacheManager nodeCacheManager;
  88     private volatile INeutronNetworkCRUD neutronNetworkCache;
  89     private volatile INeutronSubnetCRUD neutronSubnetCache;
  90     private volatile INeutronPortCRUD neutronPortCache;
  91     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  92     private volatile L3ForwardingProvider l3ForwardingProvider;
  93     private volatile InboundNatProvider inboundNatProvider;
  94     private volatile OutboundNatProvider outboundNatProvider;
  95     private volatile ArpProvider arpProvider;
  96     private volatile RoutingProvider routingProvider;
  97     private volatile GatewayMacResolver gatewayMacResolver;
  98     private volatile SecurityServicesManager securityServicesManager;
  99     private volatile IcmpEchoProvider icmpEchoProvider;
 100
 101     private class FloatIpData {
 102         // br-int of node where floating ip is associated with tenant port
 103         private final Long dpid;
 104         // patch port in br-int used to reach br-ex
 105         private final Long ofPort;
 106         // segmentation id of the net where fixed ip is instantiated
 107         private final String segId;
 108         // mac address assigned to neutron port of floating ip
 109         private final String macAddress;
 110         private final String floatingIpAddress;
 111         // ip address given to tenant vm
 112         private final String fixedIpAddress;
 113         private final String neutronRouterMac;
 114
 115         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 116                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 117             this.dpid = dpid;
 118             this.ofPort = ofPort;
 119             this.segId = segId;
 120             this.macAddress = macAddress;
 121             this.floatingIpAddress = floatingIpAddress;
 122             this.fixedIpAddress = fixedIpAddress;
 123             this.neutronRouterMac = neutronRouterMac;
 124         }
 125     }
 126
 127     private Map<String, String> networkIdToRouterMacCache;
 128     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 129     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 130
 131     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 132     private Map<String, FloatIpData> floatIpDataMapCache;
 133
 134     private String externalRouterMac;
 135     private Boolean enabled = false;
 136     private Boolean flgDistributedARPEnabled = true;
 137     private Boolean isCachePopulationDone = false;
 138     private Set<NeutronPort> portCleanupCache;
 139
 140     private Southbound southbound;
 141     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 142
 143     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 144     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 145     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 146     private static final String OWNER_FLOATING_IP = "network:floatingip";
 147     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 148
 149     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 150         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 151         this.neutronModelsDataStoreHelper = neutronHelper;
 152     }
 153
 154     private void initL3AdapterMembers() {
 155         Preconditions.checkNotNull(configurationService);
 156
 157         if (configurationService.isL3ForwardingEnabled()) {
 158             this.networkIdToRouterMacCache = new HashMap<>();
 159             this.networkIdToRouterIpListCache = new HashMap<>();
 160             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 161             this.neutronPortToDpIdCache = new HashMap<>();
 162             this.floatIpDataMapCache = new HashMap<>();
 163
 164             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 165             if (this.externalRouterMac == null) {
 166                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 167             }
 168
 169             this.enabled = true;
 170             LOG.info("OVSDB L3 forwarding is enabled");
 171             if (configurationService.isDistributedArpDisabled()) {
 172                 this.flgDistributedARPEnabled = false;
 173                 LOG.info("Distributed ARP responder is disabled");
 174             } else {
 175                 LOG.debug("Distributed ARP responder is enabled");
 176             }
 177         } else {
 178             LOG.debug("OVSDB L3 forwarding is disabled");
 179         }
 180         this.portCleanupCache = new HashSet<>();
 181     }
 182
 183     //
 184     // Callbacks from AbstractHandler
 185     //
 186     @Override
 187     public void processEvent(AbstractEvent abstractEvent) {
 188         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 189             LOG.error("Unable to process abstract event " + abstractEvent);
 190             return;
 191         }
 192         if (!this.enabled) {
 193             return;
 194         }
 195
 196         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 197         switch (ev.getAction()) {
 198             case UPDATE:
 199                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 200                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 201                 } else {
 202                     LOG.warn("Received update for an unexpected event " + ev);
 203                 }
 204                 break;
 205             case ADD:
 206                 // fall through...
 207                 // break;
 208             case DELETE:
 209                 // fall through...
 210                 // break;
 211             default:
 212                 LOG.warn("Unable to process event " + ev);
 213                 break;
 214         }
 215     }
 216
 217     //
 218     // Callbacks from GatewayMacResolverListener
 219     //
 220
 221     @Override
 222     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 223         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 224                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 225         if (!this.enabled) {
 226             return;
 227         }
 228
 229         if (macAddress == null || macAddress.getValue() == null) {
 230             // TODO: handle cases when mac is null
 231             return;
 232         }
 233
 234         //
 235         // Enqueue event so update is handled by adapter's thread
 236         //
 237         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 238     }
 239
 240     private void populateL3ForwardingCaches() {
 241         if (!this.enabled) {
 242             return;
 243         }
 244         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 245                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 246             return;
 247         }
 248         this.isCachePopulationDone = true;
 249         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 250         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 251         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 252         if(routers != null && routers.getRouter() != null && ports != null) {
 253             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 254             for( Router router : routers.getRouter()) {
 255                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 256                 if(!ports.getPort().isEmpty()) {
 257                     for( Port port : ports.getPort()) {
 258                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 259                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 260                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 261                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 262                                     , port.getMacAddress());
 263
 264                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 265                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 266                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 267                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 268                         }
 269                     }
 270                 }else {
 271                     LOG.warn("L3 Cache Population :Did not find any port information " +
 272                             "in config Data Store for router {}",router);
 273                 }
 274             }
 275         }
 276         LOG.debug("NetVirt L3 caches population is done");
 277     }
 278
 279     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 280         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 281             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 282             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 283             for(Node bridge : bridges) {
 284                 List<OvsdbTerminationPointAugmentation> interfaces
 285                         = southbound.extractTerminationPointAugmentations(bridge);
 286                 if(interfaces != null && !interfaces.isEmpty()) {
 287                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 288                             interfaces.size(), bridge.getNodeId());
 289                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 290                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 291                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 292                             Long dpId = getDpidForIntegrationBridge(bridge);
 293                             Uuid interfaceUuid = intf.getInterfaceUuid();
 294                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 295                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 296                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 297                             break;
 298                         }
 299                     }
 300                 }
 301             }
 302         }
 303         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 304     }
 305
 306     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 307         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 308         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 309         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 310             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 311                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 312                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 313                             "Fetching from data store.",neutronFloatingIP.getID());
 314                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 315                 }
 316             }
 317         }
 318         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 319         return floatIpDataMapCache.values();
 320     }
 321     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 322         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 323                 "floating id {} ",neutronFloatingId);
 324         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 325             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 326             if (neutronFloatingIP == null) {
 327                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 328                 return null;
 329             }
 330             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 331             NeutronPort neutronPortForFloatIp = null;
 332             for (NeutronPort neutronPort : neutronPorts) {
 333                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 334                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 335                     neutronPortForFloatIp = neutronPort;
 336                     break;
 337                 }
 338             }
 339
 340             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 341             if(neutronTenantPortUuid == null) {
 342                 return null;
 343             }
 344             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 345             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 346             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 347             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 348
 349             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 350             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 351                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 352             String providerSegmentationId = tenantNeutronNetwork != null ?
 353                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 354             String neutronRouterMac = tenantNeutronNetwork != null ?
 355                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 356
 357             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 358                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 359                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 360                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 361                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 362                                 "seg {} mac {} rtrMac {}",
 363                         fixedIpAddress,
 364                         floatingIpAddress,
 365                         neutronPortForFloatIp,
 366                         neutronTenantPortUuid,
 367                         providerSegmentationId,
 368                         floatingIpMac,
 369                         neutronRouterMac);
 370
 371                 return null;
 372             }
 373
 374             // get ofport for patch port in br-int
 375             final Long dpId = nodeIfPair.getLeft();
 376             final Long ofPort = findOFPortForExtPatch(dpId);
 377             if (ofPort == null) {
 378                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 379                                 "to connect floating ip to external bridge. dpid {}",
 380                         dpId);
 381                 return null;
 382             }
 383
 384             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 385                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 386             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 387
 388         }
 389         return floatIpDataMapCache.get(neutronFloatingId);
 390     }
 391     /**
 392      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 393      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 394      * subscribe so that the mac address associated to the gateway ip address is available.
 395      *
 396      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 397      */
 398     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 399         Preconditions.checkNotNull(externalRouterMacUpdate);
 400
 401         flushExistingIpRewrite();
 402         this.externalRouterMac = externalRouterMacUpdate;
 403         rebuildExistingIpRewrite();
 404     }
 405
 406     /**
 407      * Process the event.
 408      *
 409      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 410      * @param subnet An instance of NeutronSubnet object.
 411      */
 412     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 413         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 414     }
 415
 416     /**
 417      * Process the port event as a router interface event.
 418      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 419      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 420      *
 421      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 422      * @param neutronPort An instance of NeutronPort object.
 423      */
 424     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 425         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 426
 427         if (action == Action.UPDATE) {
 428             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 429             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 430             this.processSecurityGroupUpdate(neutronPort);
 431         }
 432
 433         if (!this.enabled) {
 434             return;
 435         }
 436
 437         final boolean isDelete = action == Action.DELETE;
 438
 439         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 440             if (!isDelete) {
 441                 LOG.info("Port {} is network router gateway interface, "
 442                         + "triggering gateway resolution for the attached external network", neutronPort);
 443                 this.triggerGatewayMacResolver(neutronPort);
 444             }else{
 445                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 446
 447                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 448                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 449                     // TODO support IPv6
 450                     if (externalSubnet != null &&
 451                             externalSubnet.getIpVersion() == 4) {
 452                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 453                     }
 454                 }
 455             }
 456         }
 457
 458         // Treat the port event as a router interface event if the port belongs to router. This is a
 459         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 460         //
 461         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 462             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 463
 464             if (neutronPort.getFixedIPs() != null) {
 465                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 466                     NeutronRouter_Interface neutronRouterInterface =
 467                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 468                     // id of router interface to be same as subnet
 469                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 470                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 471
 472                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 473                 }
 474             }
 475         } else {
 476             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 477             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 478             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 479             // there.
 480             //
 481             if (!isDelete && neutronPort.getFixedIPs() != null) {
 482                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 483                     NeutronRouter_Interface neutronRouterInterface =
 484                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 485                     if (neutronRouterInterface != null) {
 486                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 487                     }
 488                 }
 489             }
 490             this.updateL3ForNeutronPort(neutronPort, isDelete);
 491         }
 492     }
 493
 494     /**
 495      * Process the event.
 496      *
 497      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 498      * @param neutronRouter An instance of NeutronRouter object.
 499      */
 500     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 501         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 502     }
 503
 504     /**
 505      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 506      * delete the ports on the same subnet.
 507      *
 508      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 509      * @param neutronRouter An instance of NeutronRouter object.
 510      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 511      */
 512     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 513                                                   final NeutronRouter_Interface neutronRouterInterface,
 514                                                   Action action) {
 515         LOG.debug("Router interface {} got event {}. Subnet {}",
 516                      neutronRouterInterface.getPortUUID(),
 517                      action,
 518                      neutronRouterInterface.getSubnetUUID());
 519         if (!this.enabled) {
 520             return;
 521         }
 522
 523         final boolean isDelete = action == Action.DELETE;
 524
 525         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 526
 527         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 528         // see if they are affected by l3
 529         //
 530         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 531             boolean currPortShouldBeDeleted = false;
 532             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 533             if (isDelete) {
 534                 if (neutronPort.getFixedIPs() != null) {
 535                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 536                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 537                             currPortShouldBeDeleted = true;
 538                             break;
 539                         }
 540                     }
 541                 }
 542             }
 543             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 544         }
 545
 546         if (isDelete) {
 547             /*
 548              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 549              */
 550             this.cleanupRouterCache(neutronRouterInterface);
 551         }
 552     }
 553
 554     /**
 555      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 556      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 557      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 558      * way around) on OpenFlow Table 100.
 559      *
 560      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 561      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 562      */
 563     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 564                                              Action actionIn) {
 565         Preconditions.checkNotNull(neutronFloatingIP);
 566
 567         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 568                 neutronFloatingIP.getFixedIPAddress(),
 569                 neutronFloatingIP.getFloatingIPAddress(),
 570                 neutronFloatingIP.getFloatingNetworkUUID());
 571         if (!this.enabled) {
 572             return;
 573         }
 574
 575         Action action;
 576
 577         // Consider action to be delete if getFixedIPAddress is null
 578         //
 579         if (neutronFloatingIP.getFixedIPAddress() == null) {
 580             action = Action.DELETE;
 581         } else {
 582             action = actionIn;
 583         }
 584
 585         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 586
 587         if (action != Action.DELETE) {
 588             // must be first, as it updates floatIpDataMapCache
 589             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 590
 591             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 592             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 593         } else {
 594             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 595             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 596
 597             // must be last, as it updates floatIpDataMapCache
 598             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 599         }
 600     }
 601
 602     /**
 603      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 604      * ip, otherwise for newer one.
 605      */
 606     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 607         Preconditions.checkNotNull(neutronFloatingIP);
 608
 609         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 610         if (fid == null) {
 611             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 612                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 613             return;
 614         }
 615         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 616                                       action);
 617     }
 618
 619     /**
 620      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 621      * ip, otherwise for newer one.
 622      */
 623     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 624         Preconditions.checkNotNull(neutronFloatingIP);
 625
 626         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 627         if (fid == null) {
 628             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 629                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 630             return;
 631         }
 632         programOutboundIpRewriteStage1(fid, action);
 633     }
 634
 635     private void flushExistingIpRewrite() {
 636         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 637             programOutboundIpRewriteStage1(fid, Action.DELETE);
 638         }
 639     }
 640
 641     private void rebuildExistingIpRewrite() {
 642         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 643             programOutboundIpRewriteStage1(fid, Action.ADD);
 644         }
 645     }
 646
 647     /**
 648      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 649      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 650      * requests.
 651      */
 652     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 653         Preconditions.checkNotNull(neutronFloatingIP);
 654         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 655         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 656
 657         // find bridge Node where floating ip is configured by looking up cache for its port
 658         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 659         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 660         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 661         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 662         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 663         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 664
 665         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 666         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 667                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 668         final String providerSegmentationId = tenantNeutronNetwork != null ?
 669                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 670         final String neutronRouterMac = tenantNeutronNetwork != null ?
 671                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 672
 673         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 674                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 675                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 676                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 677             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 678                     fixedIpAddress,
 679                     floatingIpAddress,
 680                     neutronPortForFloatIp,
 681                     neutronTenantPortUuid,
 682                     providerSegmentationId,
 683                     floatingIpMac,
 684                     neutronRouterMac);
 685             return;
 686         }
 687
 688         // get ofport for patch port in br-int
 689         final Long dpId = nodeIfPair.getLeft();
 690         final Long ofPort = findOFPortForExtPatch(dpId);
 691         if (ofPort == null) {
 692             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 693                     dpId);
 694             return;
 695         }
 696
 697         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 698         //
 699         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 700                 Action.ADD)) {
 701             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 702                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 703             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 704             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 705                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 706                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 707         }
 708     }
 709
 710     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 711         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 712         if (floatIpData == null) {
 713             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 714             return;
 715         }
 716
 717         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 718                 floatIpData.floatingIpAddress, Action.DELETE)) {
 719             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 720             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 721                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 722         }
 723     }
 724
 725     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 726         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 727             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 728                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 729                 return neutronPort;
 730             }
 731         }
 732         return null;
 733     }
 734
 735     private Long findOFPortForExtPatch(Long dpId) {
 736         final String brInt = configurationService.getIntegrationBridgeName();
 737         final String brExt = configurationService.getExternalBridgeName();
 738         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 739
 740         Preconditions.checkNotNull(dpId);
 741         Preconditions.checkNotNull(portNameInt);
 742
 743         final long dpidPrimitive = dpId;
 744         for (Node node : nodeCacheManager.getBridgeNodes()) {
 745             if (dpidPrimitive == southbound.getDataPathId(node)) {
 746                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 747                         southbound.getTerminationPointOfBridge(node, portNameInt);
 748                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 749             }
 750         }
 751         return null;
 752     }
 753
 754     /**
 755      * Process the event.
 756      *
 757      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 758      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 759      */
 760     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 761         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 762     }
 763
 764     //
 765     // Callbacks from OVSDB's southbound handler
 766     //
 767     /**
 768      * Process the event.
 769      *
 770      * @param bridgeNode An instance of Node object.
 771      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 772      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 773      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 774      * object.
 775      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 776      */
 777     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 778                                      final NeutronNetwork neutronNetwork, Action action) {
 779         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 780                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 781
 782         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 783         if (action != Action.DELETE && neutronPort != null) {
 784             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 785             storePortInCleanupCache(neutronPort);
 786         }
 787
 788         if (!this.enabled) {
 789             return;
 790         }
 791
 792         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 793         final Uuid interfaceUuid = intf.getInterfaceUuid();
 794
 795         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 796                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 797
 798         if (neutronPort != null) {
 799             final String neutronPortUuid = neutronPort.getPortUUID();
 800
 801             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 802                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 803             }
 804
 805             handleNeutronPortEvent(neutronPort, action);
 806         }
 807
 808         if (action == Action.DELETE && interfaceUuid != null) {
 809             handleInterfaceEventDelete(intf, dpId);
 810         }
 811     }
 812
 813     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 814         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 815         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 816                 neutronPortUuid, dpId, interfaceUuid.getValue());
 817     }
 818
 819     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 820         // Remove entry from neutronPortToDpIdCache based on interface uuid
 821         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 822             final String currPortUuid = entry.getKey();
 823             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 824                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 825                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 826                 neutronPortToDpIdCache.remove(currPortUuid);
 827                 break;
 828             }
 829         }
 830     }
 831
 832     //
 833     // Internal helpers
 834     //
 835     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 836
 837         final String networkUUID = neutronPort.getNetworkUUID();
 838         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 839
 840         // If there is no router interface handling the networkUUID, we are done
 841         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 842             return;
 843         }
 844
 845         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 846         // router interface are handled via handleNeutronRouterInterfaceEvent.
 847         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 848             return;
 849         }
 850
 851         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 852         final String providerSegmentationId = neutronNetwork != null ?
 853                                               neutronNetwork.getProviderSegmentationID() : null;
 854         final String tenantMac = neutronPort.getMacAddress();
 855
 856         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 857             tenantMac == null || tenantMac.isEmpty()) {
 858             // done: go no further w/out all the info needed...
 859             return;
 860         }
 861
 862         final Action action = isDelete ? Action.DELETE : Action.ADD;
 863         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 864         if (nodes.isEmpty()) {
 865             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 866         }
 867         for (Node node : nodes) {
 868             final Long dpid = getDpidForIntegrationBridge(node);
 869             if (dpid == null) {
 870                 continue;
 871             }
 872             if (neutronPort.getFixedIPs() == null) {
 873                 continue;
 874             }
 875             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 876                 final String tenantIpStr = neutronIP.getIpAddress();
 877                 if (tenantIpStr.isEmpty()) {
 878                     continue;
 879                 }
 880
 881                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 882                 // still needed when routing to subnets non-local to node (bug 2076).
 883                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 884
 885                 // Configure distributed ARP responder
 886                 if (flgDistributedARPEnabled) {
 887                     // Arp rule is only needed when segmentation exists in the given node (bug 4752).
 888                     boolean arpNeeded = tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 889                     final Action actionForNode = arpNeeded ? action : Action.DELETE;
 890                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 891                 }
 892             }
 893         }
 894     }
 895
 896     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 897         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 898         /**
 899          * Get updated data and original data for the the changed. Identify the security groups that got
 900          * added and removed and call the appropriate providers for updating the flows.
 901          */
 902         try {
 903             NeutronPort originalPort = neutronPort.getOriginalPort();
 904             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 905                                                                             neutronPort.getOriginalPort());
 906             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 907                                                                               neutronPort);
 908
 909             if (null != addedGroup && !addedGroup.isEmpty()) {
 910                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 911             }
 912             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 913                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 914             }
 915
 916         } catch (Exception e) {
 917             LOG.error("Exception in processSecurityGroupUpdate", e);
 918         }
 919     }
 920
 921     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 922         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 923         if (port1 == null) {
 924             return null;
 925         }
 926         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 927         if (port2 == null) {
 928             return list1;
 929         }
 930         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 931         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 932             NeutronSecurityGroup securityGroup1 = iterator.next();
 933             for (NeutronSecurityGroup securityGroup2 :list2) {
 934                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 935                     iterator.remove();
 936                 }
 937             }
 938         }
 939         return list1;
 940     }
 941
 942     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 943                                            String macAddress, String ipStr,
 944                                            Action actionForNode) {
 945         if (actionForNode == Action.DELETE) {
 946             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 947                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 948         }
 949         if (actionForNode == Action.ADD) {
 950             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 951                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 952         }
 953
 954         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 955                                                        macAddress, ipStr, actionForNode);
 956     }
 957
 958     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 959                                              String macAddress,
 960                                              String address,
 961                                              Action actionForNode) {
 962         Status status;
 963         try {
 964             InetAddress inetAddress = InetAddress.getByName(address);
 965             status = l3ForwardingProvider == null ?
 966                      new Status(StatusCode.SUCCESS) :
 967                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 968                                                                       inetAddress, macAddress, actionForNode);
 969         } catch (UnknownHostException e) {
 970             status = new Status(StatusCode.BADREQUEST);
 971         }
 972
 973         if (status.isSuccess()) {
 974             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 975                          l3ForwardingProvider == null ? "skipped" : "programmed",
 976                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 977         } else {
 978             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 979                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 980         }
 981         return status;
 982     }
 983
 984     // --
 985
 986     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 987                                                        Boolean isDelete) {
 988         Preconditions.checkNotNull(destNeutronRouterInterface);
 989
 990         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 991         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 992         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 993         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 994         final NeutronNetwork neutronNetwork = subnet != null ?
 995                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 996         final String destinationSegmentationId = neutronNetwork != null ?
 997                                                  neutronNetwork.getProviderSegmentationID() : null;
 998         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 999         final String cidr = subnet != null ? subnet.getCidr() : null;
1000         final int mask = getMaskLenFromCidr(cidr);
1001
1002         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1003                      destNeutronRouterInterface, isDelete);
1004
1005         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1006         // the local cache
1007         if (neutronNetwork != null) {
1008             if (macAddress == null || macAddress.isEmpty()) {
1009                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1010             }
1011             if (ipList == null || ipList.isEmpty()) {
1012                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1013             }
1014         }
1015
1016         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1017             cidr == null || cidr.isEmpty() ||
1018             macAddress == null || macAddress.isEmpty() ||
1019             ipList == null || ipList.isEmpty()) {
1020             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1021                          destinationSegmentationId, cidr, macAddress, ipList);
1022             // done: go no further w/out all the info needed...
1023             return;
1024         }
1025
1026         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1027
1028         // Keep cache for finding router's mac from network uuid -- add
1029         //
1030         if (! isDelete) {
1031             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1032             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1033             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1034         }
1035
1036         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1037         if (nodes.isEmpty()) {
1038             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1039         }
1040         for (Node node : nodes) {
1041             final Long dpid = getDpidForIntegrationBridge(node);
1042             if (dpid == null) {
1043                 continue;
1044             }
1045
1046             for (Neutron_IPs neutronIP : ipList) {
1047                 final String ipStr = neutronIP.getIpAddress();
1048                 if (ipStr.isEmpty()) {
1049                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1050                             node.getNodeId().getValue(), ipStr);
1051                     continue;
1052                 }
1053
1054                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1055                 //
1056                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1057                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1058                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1059                                                               neutronNetwork, destinationSegmentationId,
1060                                                               macAddress, ipStr, mask, actionForNode,
1061                                                               true /*isReflexsive*/);
1062                 }
1063
1064                 if (! isExternal) {
1065                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1066                             actionForNode);
1067                 }
1068                 // Enable ARP responder by default, because router interface needs to be responded always.
1069                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1070                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1071             }
1072
1073             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1074             // for the external neutron networks.
1075             //
1076             {
1077                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1078                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1079             }
1080         }
1081
1082         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1083     }
1084
1085     private void programFlowForNetworkFromExternal(final Node node,
1086                                                    final Long dpid,
1087                                                    final String destinationSegmentationId,
1088                                                    final String dstMacAddress,
1089                                                    final String destIpStr,
1090                                                    final int destMask,
1091                                                    final Action actionForNode) {
1092         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1093                 dstMacAddress, destIpStr, destMask, actionForNode);
1094     }
1095
1096     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1097                                                            final Long dpid,
1098                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1099                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1100                                                            final NeutronNetwork dstNeutronNetwork,
1101                                                            final String destinationSegmentationId,
1102                                                            final String dstMacAddress,
1103                                                            final String destIpStr,
1104                                                            final int destMask,
1105                                                            final Action actionForNode,
1106                                                            Boolean isReflexsive) {
1107         Preconditions.checkNotNull(srcNeutronRouterInterface);
1108         Preconditions.checkNotNull(dstNeutronRouterInterface);
1109
1110         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1111         if (sourceSubnetId == null) {
1112             LOG.error("Could not get provider Subnet ID from router interface {}",
1113                          srcNeutronRouterInterface.getID());
1114             return;
1115         }
1116
1117         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1118         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1119         if (sourceNetworkId == null) {
1120             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1121             return;
1122         }
1123
1124         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1125         if (sourceNetwork == null) {
1126             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1127             return;
1128         }
1129
1130         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1131             // Isolate subnets from different tenants within the same router
1132             return;
1133         }
1134         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1135         if (sourceSegmentationId == null) {
1136             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1137             return;
1138         }
1139         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1140             // Skip 'self'
1141             return;
1142         }
1143
1144         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1145                                      dstMacAddress, destIpStr, destMask, actionForNode);
1146
1147         // Flip roles src->dst; dst->src
1148         if (isReflexsive) {
1149             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1150             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1151             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1152             final String cidr2 = sourceSubnet.getCidr();
1153             final int mask2 = getMaskLenFromCidr(cidr2);
1154
1155             if (cidr2 == null || cidr2.isEmpty() ||
1156                 macAddress2 == null || macAddress2.isEmpty() ||
1157                 ipList2 == null || ipList2.isEmpty()) {
1158                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1159                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1160                 // done: go no further w/out all the info needed...
1161                 return;
1162             }
1163
1164             for (Neutron_IPs neutronIP2 : ipList2) {
1165                 final String ipStr2 = neutronIP2.getIpAddress();
1166                 if (ipStr2.isEmpty()) {
1167                     continue;
1168                 }
1169                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1170                                                           srcNeutronRouterInterface,
1171                                                           sourceNetwork, sourceSegmentationId,
1172                                                           macAddress2, ipStr2, mask2, actionForNode,
1173                                                           false /*isReflexsive*/);
1174             }
1175         }
1176     }
1177
1178     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1179                                               String destinationSegmentationId,
1180                                               String macAddress, String ipStr, int mask,
1181                                               Action actionForNode) {
1182         if (actionForNode == Action.DELETE) {
1183             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1184                          " action {}",
1185                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1186                          macAddress, ipStr, mask, actionForNode);
1187         }
1188         if (actionForNode == Action.ADD) {
1189             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1190                          " action {}",
1191                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1192                          macAddress, ipStr, mask, actionForNode);
1193         }
1194
1195         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1196                                                           macAddress, ipStr, mask, actionForNode);
1197     }
1198
1199     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1200                                                 String destinationSegmentationId,
1201                                                 String macAddress,
1202                                                 String address, int mask,
1203                                                 Action actionForNode) {
1204         Status status;
1205         try {
1206             InetAddress inetAddress = InetAddress.getByName(address);
1207             status = routingProvider == null ?
1208                      new Status(StatusCode.SUCCESS) :
1209                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1210                                                             macAddress, inetAddress, mask, actionForNode);
1211         } catch (UnknownHostException e) {
1212             status = new Status(StatusCode.BADREQUEST);
1213         }
1214
1215         if (status.isSuccess()) {
1216             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1217                          routingProvider == null ? "skipped" : "programmed",
1218                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1219                          actionForNode);
1220         } else {
1221             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1222                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1223                          actionForNode, status);
1224         }
1225         return status;
1226     }
1227
1228     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1229                                            String macAddress, String ipStr,
1230                                            Action action) {
1231         if (action == Action.DELETE ) {
1232             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1233                     dpid, segOrOfPort, macAddress, ipStr, action);
1234         }
1235         if (action == Action.ADD) {
1236             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1237                     dpid, segOrOfPort, macAddress, ipStr, action);
1238         }
1239
1240         Status status = new Status(StatusCode.UNSUPPORTED);
1241         if (icmpEchoProvider != null){
1242             try {
1243                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1244                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1245                                                 macAddress, inetAddress, action);
1246             } catch (UnknownHostException e) {
1247                 status = new Status(StatusCode.BADREQUEST);
1248             }
1249         }
1250
1251         if (status.isSuccess()) {
1252             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1253                     icmpEchoProvider == null ? "skipped" : "programmed",
1254                     macAddress, ipStr, dpid, segOrOfPort, action);
1255         } else {
1256             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1257                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1258         }
1259
1260         return status.isSuccess();
1261     }
1262
1263     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1264                                            String macAddress, String ipStr,
1265                                            Action action) {
1266         if (action == Action.DELETE ) {
1267             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1268                     dpid, segOrOfPort, macAddress, ipStr, action);
1269         }
1270         if (action == Action.ADD) {
1271             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1272                     dpid, segOrOfPort, macAddress, ipStr, action);
1273         }
1274
1275         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1276         return status.isSuccess();
1277     }
1278
1279     private Status programStaticArpStage2(Long dpid,
1280                                           String segOrOfPort,
1281                                           String macAddress,
1282                                           String address,
1283                                           Action action) {
1284         Status status;
1285         try {
1286             InetAddress inetAddress = InetAddress.getByName(address);
1287             status = arpProvider == null ?
1288                      new Status(StatusCode.SUCCESS) :
1289                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1290                                                        macAddress, inetAddress, action);
1291         } catch (UnknownHostException e) {
1292             status = new Status(StatusCode.BADREQUEST);
1293         }
1294
1295         if (status.isSuccess()) {
1296             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1297                          arpProvider == null ? "skipped" : "programmed",
1298                          macAddress, address, dpid, segOrOfPort, action);
1299         } else {
1300             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1301                          macAddress, address, dpid, segOrOfPort, action, status);
1302         }
1303         return status;
1304     }
1305
1306     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1307                                                   String matchAddress, String rewriteAddress,
1308                                                   Action action) {
1309         if (action == Action.DELETE ) {
1310             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1311                     " action {}",
1312                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1313         }
1314         if (action == Action.ADD ) {
1315             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1316                     " action {}",
1317                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1318         }
1319
1320         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1321                 rewriteAddress, action);
1322         return status.isSuccess();
1323     }
1324
1325     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1326                                                  String matchAddress, String rewriteAddress,
1327                                                  Action action) {
1328         Status status;
1329         try {
1330             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1331             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1332             status = inboundNatProvider == null ?
1333                     new Status(StatusCode.SUCCESS) :
1334                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1335                             inetMatchAddress, inetRewriteAddress,
1336                             action);
1337         } catch (UnknownHostException e) {
1338             status = new Status(StatusCode.BADREQUEST);
1339         }
1340
1341         if (status.isSuccess()) {
1342             final boolean isSkipped = inboundNatProvider == null;
1343             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1344                     isSkipped ? "skipped" : "programmed",
1345                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1346         } else {
1347             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1348                          " status:{}",
1349                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1350                     status);
1351         }
1352         return status;
1353     }
1354
1355     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1356                                                  Action actionForRewriteExclusion) {
1357         if (actionForRewriteExclusion == Action.DELETE ) {
1358             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1359                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1360         }
1361         if (actionForRewriteExclusion == Action.ADD) {
1362             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1363                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1364         }
1365
1366         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1367     }
1368
1369     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1370                                                    Action actionForNode) {
1371         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1372                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1373
1374         if (status.isSuccess()) {
1375             final boolean isSkipped = outboundNatProvider == null;
1376             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1377                          isSkipped ? "skipped" : "programmed",
1378                          cidr, node.getNodeId().getValue(), actionForNode);
1379         } else {
1380             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1381                          cidr, node.getNodeId().getValue(), actionForNode, status);
1382         }
1383         return status;
1384     }
1385
1386     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1387
1388         if (action == Action.DELETE) {
1389             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1390                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1391         }
1392         if (action == Action.ADD) {
1393             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1394                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1395         }
1396
1397         this.programOutboundIpRewriteStage2(fid, action);
1398     }
1399
1400     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1401         Status status;
1402         try {
1403             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1404             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1405             status = outboundNatProvider == null ?
1406                     new Status(StatusCode.SUCCESS) :
1407                     outboundNatProvider.programIpRewriteRule(
1408                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1409                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1410         } catch (UnknownHostException e) {
1411             status = new Status(StatusCode.BADREQUEST);
1412         }
1413
1414         if (status.isSuccess()) {
1415             final boolean isSkipped = outboundNatProvider == null;
1416             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1417                             " action {}",
1418                          isSkipped ? "skipped" : "programmed",
1419                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1420         } else {
1421             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1422                          " action {} status:{}",
1423                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1424         }
1425         return status;
1426     }
1427
1428     private int getMaskLenFromCidr(String cidr) {
1429         if (cidr == null) {
1430             return 0;
1431         }
1432         String[] splits = cidr.split("/");
1433         if (splits.length != 2) {
1434             return 0;
1435         }
1436
1437         int result;
1438         try {
1439             result = Integer.parseInt(splits[1].trim());
1440         } catch (NumberFormatException nfe) {
1441             result = 0;
1442         }
1443         return result;
1444     }
1445
1446     private Long getDpidForIntegrationBridge(Node node) {
1447         // Check if node is integration bridge; and only then return its dpid
1448         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1449             return southbound.getDataPathId(node);
1450         }
1451         return null;
1452     }
1453
1454     private Long getDpidForExternalBridge(Node node) {
1455         // Check if node is external bridge; and only then return its dpid
1456         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1457             return southbound.getDataPathId(node);
1458         }
1459         return null;
1460     }
1461
1462     private Node getExternalBridgeNode(){
1463         //Pickup the first node that has external bridge (br-ex).
1464         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1465         //the external network is reachable from every br-ex
1466         // TODO: Consider other deployment scenario, and thing of better solution.
1467         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1468         for(Node node : allBridges){
1469             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1470                 return node;
1471             }
1472         }
1473         return null;
1474     }
1475
1476     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1477         if (gatewayPort.getFixedIPs() == null) {
1478             return null;
1479         }
1480         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1481             String subnetUUID = neutronIPs.getSubnetUUID();
1482             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1483             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1484                 return extSubnet;
1485             }
1486             if (extSubnet == null) {
1487                 // TODO: when subnet is created, try again.
1488                 LOG.debug("subnet {} in not found", subnetUUID);
1489              }
1490         }
1491         return null;
1492     }
1493
1494      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1495          /*
1496           *  Fix for 4277
1497           *  Remove the router cache only after deleting the neutron
1498           *  port l3 flows.
1499           */
1500          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1501
1502          if (neutronPort != null) {
1503              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1504              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1505              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1506          }
1507      }
1508
1509     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1510
1511         Preconditions.checkNotNull(gatewayPort);
1512         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1513
1514         if(externalNetwork != null){
1515             if(externalNetwork.isRouterExternal()){
1516                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1517
1518                 // TODO: address IPv6 case.
1519                 if (externalSubnet != null &&
1520                         externalSubnet.getIpVersion() == 4 &&
1521                         gatewayPort.getFixedIPs() != null) {
1522                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1523
1524                     gatewayMacResolver.resolveMacAddress(
1525                             this, /* gatewayMacResolverListener */
1526                             null, /* externalNetworkBridgeDpid */
1527                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1528                             new Ipv4Address(externalSubnet.getGatewayIP()),
1529                             new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1530                             new MacAddress(gatewayPort.getMacAddress()),
1531                             true /* periodicRefresh */);
1532                 } else {
1533                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1534                 }
1535             }
1536         }else{
1537             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1538         }
1539     }
1540
1541
1542     private void storePortInCleanupCache(NeutronPort port) {
1543         this.portCleanupCache.add(port);
1544     }
1545
1546
1547     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1548         removePortFromCleanupCache(originalPort);
1549         storePortInCleanupCache(updatedPort);
1550     }
1551
1552     public void removePortFromCleanupCache(NeutronPort port) {
1553         this.portCleanupCache.remove(port);
1554     }
1555
1556     public NeutronPort getPortFromCleanupCache(String portid) {
1557         for (NeutronPort neutronPort : this.portCleanupCache) {
1558             if (neutronPort.getPortUUID() != null ) {
1559                 if (neutronPort.getPortUUID().equals(portid)) {
1560                     LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1561                     return neutronPort;
1562                     }
1563                 }
1564             }
1565         return null;
1566     }
1567
1568     /**
1569      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1570      *
1571      * @param ofPort the OF port number
1572      * @return the string with encoded OF port (example format "OFPort|999")
1573      */
1574     public static String encodeExcplicitOFPort(Long ofPort) {
1575         return "OFPort|" + ofPort.toString();
1576     }
1577
1578     @Override
1579     public void setDependencies(ServiceReference serviceReference) {
1580         eventDispatcher =
1581                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1582         eventDispatcher.eventHandlerAdded(serviceReference, this);
1583         tenantNetworkManager =
1584                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1585         configurationService =
1586                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1587         arpProvider =
1588                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1589         inboundNatProvider =
1590                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1591         outboundNatProvider =
1592                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1593         routingProvider =
1594                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1595         l3ForwardingProvider =
1596                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1597         nodeCacheManager =
1598                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1599         southbound =
1600                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1601         gatewayMacResolver =
1602                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1603         securityServicesManager =
1604                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1605
1606         initL3AdapterMembers();
1607     }
1608
1609     @Override
1610     public void setDependencies(Object impl) {
1611         if (impl instanceof INeutronNetworkCRUD) {
1612             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1613         } else if (impl instanceof INeutronPortCRUD) {
1614             neutronPortCache = (INeutronPortCRUD)impl;
1615         } else if (impl instanceof INeutronSubnetCRUD) {
1616             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1617         } else if (impl instanceof INeutronFloatingIPCRUD) {
1618             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1619         } else if (impl instanceof ArpProvider) {
1620             arpProvider = (ArpProvider)impl;
1621         } else if (impl instanceof InboundNatProvider) {
1622             inboundNatProvider = (InboundNatProvider)impl;
1623         } else if (impl instanceof OutboundNatProvider) {
1624             outboundNatProvider = (OutboundNatProvider)impl;
1625         } else if (impl instanceof RoutingProvider) {
1626             routingProvider = (RoutingProvider)impl;
1627         } else if (impl instanceof L3ForwardingProvider) {
1628             l3ForwardingProvider = (L3ForwardingProvider)impl;
1629         }else if (impl instanceof GatewayMacResolver) {
1630             gatewayMacResolver = (GatewayMacResolver)impl;
1631         }else if (impl instanceof IcmpEchoProvider) {
1632             icmpEchoProvider = (IcmpEchoProvider)impl;
1633         }
1634
1635         populateL3ForwardingCaches();
1636     }
1637 }