openstack/net-virt/src/main/java/org/opendaylight/netvirt/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright © 2014, 2017 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.openstack.netvirt.impl;
  10
  11 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.ADD;
  12 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.DELETE;
  13 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.UPDATE;
  14
  15 import com.google.common.base.Preconditions;
  16 import org.apache.commons.lang3.tuple.ImmutablePair;
  17 import org.apache.commons.lang3.tuple.Pair;
  18 import org.opendaylight.netvirt.openstack.netvirt.AbstractEvent;
  19 import org.opendaylight.netvirt.openstack.netvirt.AbstractHandler;
  20 import org.opendaylight.netvirt.openstack.netvirt.ConfigInterface;
  21 import org.opendaylight.netvirt.openstack.netvirt.NeutronL3AdapterEvent;
  22 import org.opendaylight.netvirt.openstack.netvirt.NeutronModelsDataStoreHelper;
  23 import org.opendaylight.netvirt.openstack.netvirt.api.Action;
  24 import org.opendaylight.netvirt.openstack.netvirt.api.ArpProvider;
  25 import org.opendaylight.netvirt.openstack.netvirt.api.ConfigurationService;
  26 import org.opendaylight.netvirt.openstack.netvirt.api.Constants;
  27 import org.opendaylight.netvirt.openstack.netvirt.api.EventDispatcher;
  28 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolver;
  29 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolverListener;
  30 import org.opendaylight.netvirt.openstack.netvirt.api.IcmpEchoProvider;
  31 import org.opendaylight.netvirt.openstack.netvirt.api.InboundNatProvider;
  32 import org.opendaylight.netvirt.openstack.netvirt.api.L3ForwardingProvider;
  33 import org.opendaylight.netvirt.openstack.netvirt.api.NodeCacheManager;
  34 import org.opendaylight.netvirt.openstack.netvirt.api.OutboundNatProvider;
  35 import org.opendaylight.netvirt.openstack.netvirt.api.RoutingProvider;
  36 import org.opendaylight.netvirt.openstack.netvirt.api.SecurityServicesManager;
  37 import org.opendaylight.netvirt.openstack.netvirt.api.Southbound;
  38 import org.opendaylight.netvirt.openstack.netvirt.api.Status;
  39 import org.opendaylight.netvirt.openstack.netvirt.api.StatusCode;
  40 import org.opendaylight.netvirt.openstack.netvirt.api.TenantNetworkManager;
  41 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronFloatingIP;
  42 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronNetwork;
  43 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter;
  44 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter_Interface;
  45 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSecurityGroup;
  46 import org.opendaylight.netvirt.openstack.netvirt.translator.Neutron_IPs;
  47 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  48 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  49 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronPortCRUD;
  50 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronPort;
  51 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSubnet;
  52 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  53 import org.opendaylight.netvirt.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  54 import org.opendaylight.netvirt.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.port.attributes.FixedIps;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  65 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  66 import org.osgi.framework.ServiceReference;
  67 import org.slf4j.Logger;
  68 import org.slf4j.LoggerFactory;
  69
  70 import java.net.Inet4Address;
  71 import java.net.InetAddress;
  72 import java.net.UnknownHostException;
  73 import java.util.ArrayList;
  74 import java.util.Collection;
  75 import java.util.HashMap;
  76 import java.util.Iterator;
  77 import java.util.List;
  78 import java.util.Map;
  79 import java.util.concurrent.ConcurrentHashMap;
  80 import java.util.concurrent.ConcurrentMap;
  81
  82 /**
  83  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  84  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  85  * as well as the multi-tenant router forwarding provider.
  86  */
  87 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  88     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  89
  90     // The implementation for each of these services is resolved by the OSGi Service Manager
  91     private volatile ConfigurationService configurationService;
  92     private volatile TenantNetworkManager tenantNetworkManager;
  93     private volatile NodeCacheManager nodeCacheManager;
  94     private volatile INeutronNetworkCRUD neutronNetworkCache;
  95     private volatile INeutronSubnetCRUD neutronSubnetCache;
  96     private volatile INeutronPortCRUD neutronPortCache;
  97     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  98     private volatile L3ForwardingProvider l3ForwardingProvider;
  99     private volatile InboundNatProvider inboundNatProvider;
 100     private volatile OutboundNatProvider outboundNatProvider;
 101     private volatile ArpProvider arpProvider;
 102     private volatile RoutingProvider routingProvider;
 103     private volatile GatewayMacResolver gatewayMacResolver;
 104     private volatile SecurityServicesManager securityServicesManager;
 105     private volatile IcmpEchoProvider icmpEchoProvider;
 106
 107     private class FloatIpData {
 108         // br-int of node where floating ip is associated with tenant port
 109         private final Long dpid;
 110         // patch port in br-int used to reach br-ex
 111         private final Long ofPort;
 112         // segmentation id of the net where fixed ip is instantiated
 113         private final String segId;
 114         // mac address assigned to neutron port of floating ip
 115         private final String macAddress;
 116         private final String floatingIpAddress;
 117         // ip address given to tenant vm
 118         private final String fixedIpAddress;
 119         private final String neutronRouterMac;
 120
 121         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 122                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 123             this.dpid = dpid;
 124             this.ofPort = ofPort;
 125             this.segId = segId;
 126             this.macAddress = macAddress;
 127             this.floatingIpAddress = floatingIpAddress;
 128             this.fixedIpAddress = fixedIpAddress;
 129             this.neutronRouterMac = neutronRouterMac;
 130         }
 131     }
 132
 133     private Map<String, String> networkIdToRouterMacCache;
 134     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 135     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 136
 137     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 138     private Map<String, FloatIpData> floatIpDataMapCache;
 139
 140     private String externalRouterMac;
 141     private Boolean enabled = false;
 142     private Boolean isCachePopulationDone = false;
 143     private ConcurrentMap<String, NeutronPort> portCleanupCache;
 144     private ConcurrentMap<String, NeutronNetwork> networkCleanupCache;
 145
 146     private Southbound southbound;
 147     private DistributedArpService distributedArpService;
 148     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 149
 150     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 151     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 152     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 153     private static final String OWNER_FLOATING_IP = "network:floatingip";
 154     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 155
 156     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 157         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 158         this.neutronModelsDataStoreHelper = neutronHelper;
 159     }
 160
 161     private void initL3AdapterMembers() {
 162         Preconditions.checkNotNull(configurationService);
 163
 164         if (configurationService.isL3ForwardingEnabled()) {
 165             this.networkIdToRouterMacCache = new HashMap<>();
 166             this.networkIdToRouterIpListCache = new HashMap<>();
 167             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 168             this.neutronPortToDpIdCache = new HashMap<>();
 169             this.floatIpDataMapCache = new HashMap<>();
 170
 171             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 172             if (this.externalRouterMac == null) {
 173                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 174             }
 175             this.enabled = true;
 176             LOG.info("OVSDB L3 forwarding is enabled");
 177         } else {
 178             LOG.debug("OVSDB L3 forwarding is disabled");
 179         }
 180         this.portCleanupCache = new ConcurrentHashMap<>();
 181         this.networkCleanupCache = new ConcurrentHashMap<>();
 182     }
 183
 184     //
 185     // Callbacks from AbstractHandler
 186     //
 187     @Override
 188     public void processEvent(AbstractEvent abstractEvent) {
 189         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 190             LOG.error("Unable to process abstract event " + abstractEvent);
 191             return;
 192         }
 193         if (!this.enabled) {
 194             return;
 195         }
 196
 197         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 198         switch (ev.getAction()) {
 199             case UPDATE:
 200                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 201                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 202                 } else {
 203                     LOG.warn("Received update for an unexpected event " + ev);
 204                 }
 205                 break;
 206             case ADD:
 207                 // fall through...
 208                 // break;
 209             case DELETE:
 210                 // fall through...
 211                 // break;
 212             default:
 213                 LOG.warn("Unable to process event " + ev);
 214                 break;
 215         }
 216     }
 217
 218     //
 219     // Callbacks from GatewayMacResolverListener
 220     //
 221
 222     @Override
 223     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 224         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 225                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 226         if (!this.enabled) {
 227             return;
 228         }
 229
 230         if (macAddress == null || macAddress.getValue() == null) {
 231             // TODO: handle cases when mac is null
 232             return;
 233         }
 234
 235         //
 236         // Enqueue event so update is handled by adapter's thread
 237         //
 238         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 239     }
 240
 241     private void populateL3ForwardingCaches() {
 242         if (!this.enabled) {
 243             return;
 244         }
 245         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 246                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 247             return;
 248         }
 249         this.isCachePopulationDone = true;
 250         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 251         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 252         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 253         if(routers != null && routers.getRouter() != null && ports != null) {
 254             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 255             for( Router router : routers.getRouter()) {
 256                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 257                 if(!ports.getPort().isEmpty()) {
 258                     for( Port port : ports.getPort()) {
 259                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 260                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 261                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 262                             for (final FixedIps fixedIps : port.getFixedIps()) {
 263                                 if (fixedIps.getIpAddress().getIpv4Address() != null) {
 264                                     networkIdToRouterMacCache.put(port.getNetworkId().getValue(),
 265                                                                   port.getMacAddress().getValue());
 266                                     networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 267                                                                      NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 268                                     subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 269                                                                        NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 270                                     break;
 271                                 }
 272                             }
 273                         }
 274                     }
 275                 }else {
 276                     LOG.warn("L3 Cache Population :Did not find any port information " +
 277                             "in config Data Store for router {}",router);
 278                 }
 279             }
 280         }
 281         LOG.debug("NetVirt L3 caches population is done");
 282     }
 283
 284     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 285         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 286             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 287             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 288             for(Node bridge : bridges) {
 289                 List<OvsdbTerminationPointAugmentation> interfaces
 290                         = southbound.extractTerminationPointAugmentations(bridge);
 291                 if(interfaces != null && !interfaces.isEmpty()) {
 292                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 293                             interfaces.size(), bridge.getNodeId());
 294                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 295                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 296                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 297                             Long dpId = getDpidForIntegrationBridge(bridge);
 298                             Uuid interfaceUuid = intf.getInterfaceUuid();
 299                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 300                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 301                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 302                             break;
 303                         }
 304                     }
 305                 }
 306             }
 307         }
 308         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 309     }
 310
 311     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 312         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 313         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 314         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 315             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 316                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 317                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 318                             "Fetching from data store.",neutronFloatingIP.getID());
 319                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 320                 }
 321             }
 322         }
 323         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 324         return floatIpDataMapCache.values();
 325     }
 326     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 327         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 328                 "floating id {} ",neutronFloatingId);
 329         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 330             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 331             if (neutronFloatingIP == null) {
 332                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 333                 return null;
 334             }
 335             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 336             NeutronPort neutronPortForFloatIp = null;
 337             for (NeutronPort neutronPort : neutronPorts) {
 338                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 339                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 340                     neutronPortForFloatIp = neutronPort;
 341                     break;
 342                 }
 343             }
 344
 345             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 346             if(neutronTenantPortUuid == null) {
 347                 return null;
 348             }
 349             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 350             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 351             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 352             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 353
 354             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 355             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 356                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 357             String providerSegmentationId = tenantNeutronNetwork != null ?
 358                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 359             String neutronRouterMac = tenantNeutronNetwork != null ?
 360                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 361
 362             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 363                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 364                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 365                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 366                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 367                                 "seg {} mac {} rtrMac {}",
 368                         fixedIpAddress,
 369                         floatingIpAddress,
 370                         neutronPortForFloatIp,
 371                         neutronTenantPortUuid,
 372                         providerSegmentationId,
 373                         floatingIpMac,
 374                         neutronRouterMac);
 375
 376                 return null;
 377             }
 378
 379             // get ofport for patch port in br-int
 380             final Long dpId = nodeIfPair.getLeft();
 381             final Long ofPort = findOFPortForExtPatch(dpId);
 382             if (ofPort == null) {
 383                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 384                                 "to connect floating ip to external bridge. dpid {}",
 385                         dpId);
 386                 return null;
 387             }
 388
 389             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 390                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 391             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 392
 393         }
 394         return floatIpDataMapCache.get(neutronFloatingId);
 395     }
 396     /**
 397      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 398      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 399      * subscribe so that the mac address associated to the gateway ip address is available.
 400      *
 401      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 402      */
 403     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 404         Preconditions.checkNotNull(externalRouterMacUpdate);
 405
 406         flushExistingIpRewrite();
 407         this.externalRouterMac = externalRouterMacUpdate;
 408         rebuildExistingIpRewrite();
 409     }
 410
 411     /**
 412      * Process the event.
 413      *
 414      * @param action the {@link Action} action to be handled.
 415      * @param subnet An instance of NeutronSubnet object.
 416      */
 417     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 418         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 419         if (action == ADD) {
 420             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 421         }
 422     }
 423
 424     /**
 425      * Process the port event as a router interface event.
 426      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 427      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 428      *
 429      * @param action the {@link Action} action to be handled.
 430      * @param neutronPort An instance of NeutronPort object.
 431      */
 432     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 433         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 434
 435         if (action == UPDATE) {
 436             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 437             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 438             if (neutronPort.getPortSecurityEnabled()) {
 439                 this.processSecurityGroupUpdate(neutronPort);
 440             }
 441             if (isPortSecurityEnableUpdated(neutronPort)) {
 442                 this.processPortSecurityEnableUpdated(neutronPort);
 443             }
 444         }
 445
 446         if (!this.enabled) {
 447             return;
 448         }
 449
 450         final boolean isDelete = action == DELETE;
 451
 452         if (action == DELETE) {
 453             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 454             this.cleanupFloatingIPRules(neutronPort);
 455         }
 456         else if (action == UPDATE){
 457             // Bug 5353: VM restart cause floatingIp flows to be removed
 458             this.updateFloatingIPRules(neutronPort);
 459         }
 460
 461         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 462             if (!isDelete) {
 463                 LOG.info("Port {} is network router gateway interface, "
 464                         + "triggering gateway resolution for the attached external network", neutronPort);
 465                 this.triggerGatewayMacResolver(neutronPort);
 466             }else{
 467                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 468                 if (null == externalNetwork) {
 469                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 470                 }
 471
 472                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 473                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 474                     // TODO support IPv6
 475                     if (externalSubnet != null &&
 476                             externalSubnet.getIpVersion() == 4) {
 477                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 478                     }
 479                 }
 480             }
 481         }
 482
 483         // Treat the port event as a router interface event if the port belongs to router. This is a
 484         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 485         //
 486         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 487             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 488
 489             if (neutronPort.getFixedIPs() != null) {
 490                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 491                     NeutronRouter_Interface neutronRouterInterface =
 492                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 493                     // id of router interface to be same as subnet
 494                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 495                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 496
 497                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 498                 }
 499             }
 500         } else {
 501             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 502             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 503             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 504             // there.
 505             //
 506             if (!isDelete && neutronPort.getFixedIPs() != null) {
 507                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 508                     NeutronRouter_Interface neutronRouterInterface =
 509                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 510                     if (neutronRouterInterface != null) {
 511                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 512                     }
 513                 }
 514             }
 515             this.updateL3ForNeutronPort(neutronPort, isDelete);
 516         }
 517     }
 518
 519     /**
 520      * Process the event.
 521      *
 522      * @param action the {@link Action} action to be handled.
 523      * @param neutronRouter An instance of NeutronRouter object.
 524      */
 525     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 526         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 527     }
 528
 529     /**
 530      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 531      * delete the ports on the same subnet.
 532      *
 533      * @param action the {@link Action} action to be handled.
 534      * @param neutronRouter An instance of NeutronRouter object.
 535      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 536      */
 537     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 538                                                   final NeutronRouter_Interface neutronRouterInterface,
 539                                                   Action action) {
 540         LOG.debug("Router interface {} got event {}. Subnet {}",
 541                      neutronRouterInterface.getPortUUID(),
 542                      action,
 543                      neutronRouterInterface.getSubnetUUID());
 544         if (!this.enabled) {
 545             return;
 546         }
 547
 548         final boolean isDelete = action == DELETE;
 549
 550         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 551
 552         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 553         // see if they are affected by l3
 554         //
 555         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 556             boolean currPortShouldBeDeleted = false;
 557             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 558             if (isDelete) {
 559                 if (neutronPort.getFixedIPs() != null) {
 560                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 561                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 562                             currPortShouldBeDeleted = true;
 563                             break;
 564                         }
 565                     }
 566                 }
 567             }
 568             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 569         }
 570
 571     }
 572
 573     /**
 574      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 575      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 576      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 577      * way around) on OpenFlow Table 100.
 578      *
 579      * @param actionIn the {@link Action} action to be handled.
 580      * @param neutronFloatingIP An {@link NeutronFloatingIP} instance of NeutronFloatingIP object.
 581      */
 582     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 583                                              Action actionIn) {
 584         Preconditions.checkNotNull(neutronFloatingIP);
 585
 586         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 587                 neutronFloatingIP.getFixedIPAddress(),
 588                 neutronFloatingIP.getFloatingIPAddress(),
 589                 neutronFloatingIP.getFloatingNetworkUUID());
 590         if (!this.enabled) {
 591             return;
 592         }
 593
 594         Action action;
 595
 596         // Consider action to be delete if getFixedIPAddress is null
 597         //
 598         if (neutronFloatingIP.getFixedIPAddress() == null) {
 599             action = DELETE;
 600         } else {
 601             action = actionIn;
 602         }
 603
 604         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 605
 606         if (action != DELETE) {
 607             // must be first, as it updates floatIpDataMapCache
 608             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 609
 610             programFlowsForFloatingIPInbound(neutronFloatingIP, ADD);
 611             programFlowsForFloatingIPOutbound(neutronFloatingIP, ADD);
 612         } else {
 613             programFlowsForFloatingIPOutbound(neutronFloatingIP, DELETE);
 614             programFlowsForFloatingIPInbound(neutronFloatingIP, DELETE);
 615
 616             // must be last, as it updates floatIpDataMapCache
 617             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 618         }
 619     }
 620
 621     /**
 622      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 623      * ip, otherwise for newer one.
 624      */
 625     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 626         Preconditions.checkNotNull(neutronFloatingIP);
 627
 628         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 629         if (fid == null) {
 630             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 631                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 632             return;
 633         }
 634         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 635                                       action);
 636     }
 637
 638     /**
 639      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 640      * ip, otherwise for newer one.
 641      */
 642     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 643         Preconditions.checkNotNull(neutronFloatingIP);
 644
 645         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 646         if (fid == null) {
 647             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 648                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 649             return;
 650         }
 651         programOutboundIpRewriteStage1(fid, action);
 652     }
 653
 654     private void flushExistingIpRewrite() {
 655         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 656             programOutboundIpRewriteStage1(fid, DELETE);
 657         }
 658     }
 659
 660     private void rebuildExistingIpRewrite() {
 661         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 662             programOutboundIpRewriteStage1(fid, ADD);
 663         }
 664     }
 665
 666     /**
 667      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 668      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 669      * requests.
 670      */
 671     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 672         Preconditions.checkNotNull(neutronFloatingIP);
 673         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 674         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 675
 676         // find bridge Node where floating ip is configured by looking up cache for its port
 677         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 678         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 679         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 680         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 681         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 682         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 683
 684         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 685         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 686                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 687         final String providerSegmentationId = tenantNeutronNetwork != null ?
 688                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 689         final String neutronRouterMac = tenantNeutronNetwork != null ?
 690                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 691
 692         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 693                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 694                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 695                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 696             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 697                     fixedIpAddress,
 698                     floatingIpAddress,
 699                     neutronPortForFloatIp,
 700                     neutronTenantPortUuid,
 701                     providerSegmentationId,
 702                     floatingIpMac,
 703                     neutronRouterMac);
 704             return;
 705         }
 706
 707         // get ofport for patch port in br-int
 708         final Long dpId = nodeIfPair.getLeft();
 709         final Long ofPort = findOFPortForExtPatch(dpId);
 710         if (ofPort == null) {
 711             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 712                     dpId);
 713             return;
 714         }
 715
 716         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 717         //
 718         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 719                 ADD)) {
 720             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 721                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 722             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 723             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 724                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 725                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 726         }
 727     }
 728
 729     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 730         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 731         if (floatIpData == null) {
 732             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 733             return;
 734         }
 735
 736         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 737                 floatIpData.floatingIpAddress, DELETE)) {
 738             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 739             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 740                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 741         }
 742     }
 743
 744     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 745         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 746             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 747                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 748                 return neutronPort;
 749             }
 750         }
 751         return null;
 752     }
 753
 754     private Long findOFPortForExtPatch(Long dpId) {
 755         final String brInt = configurationService.getIntegrationBridgeName();
 756         final String brExt = configurationService.getExternalBridgeName();
 757         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 758
 759         Preconditions.checkNotNull(dpId);
 760         Preconditions.checkNotNull(portNameInt);
 761
 762         final long dpidPrimitive = dpId;
 763         for (Node node : nodeCacheManager.getBridgeNodes()) {
 764             if (dpidPrimitive == southbound.getDataPathId(node)) {
 765                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 766                         southbound.getTerminationPointOfBridge(node, portNameInt);
 767                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 768             }
 769         }
 770         return null;
 771     }
 772
 773     /**
 774      * Process the event.
 775      *
 776      * @param action the {@link Action} action to be handled.
 777      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronFloatingIP object.
 778      */
 779     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 780         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 781         if (action == UPDATE) {
 782             this.updateNetworkInCleanupCache(neutronNetwork);
 783         }
 784     }
 785
 786     //
 787     // Callbacks from OVSDB's southbound handler
 788     //
 789     /**
 790      * Process the event.
 791      *
 792      * @param bridgeNode An instance of Node object.
 793      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 794      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 795      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronNetwork
 796      * object.
 797      * @param action the {@link Action} action to be handled.
 798      */
 799     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 800                                      final NeutronNetwork neutronNetwork, Action action) {
 801         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 802                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 803
 804         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 805         if (action != DELETE && neutronPort != null) {
 806             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 807             storePortInCleanupCache(neutronPort);
 808         }
 809
 810         if (!this.enabled) {
 811             return;
 812         }
 813
 814         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 815         final Uuid interfaceUuid = intf.getInterfaceUuid();
 816
 817         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 818                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 819
 820         if (neutronPort != null) {
 821             final String neutronPortUuid = neutronPort.getPortUUID();
 822
 823             if (action != DELETE && dpId != null && interfaceUuid != null) {
 824                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 825             }
 826
 827             handleNeutronPortEvent(neutronPort, action);
 828         }
 829
 830         if (action == DELETE && interfaceUuid != null) {
 831             handleInterfaceEventDelete(intf, dpId);
 832         }
 833     }
 834
 835     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 836         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 837         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 838                 neutronPortUuid, dpId, interfaceUuid.getValue());
 839     }
 840
 841     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 842         // Remove entry from neutronPortToDpIdCache based on interface uuid
 843         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 844             final String currPortUuid = entry.getKey();
 845             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 846                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 847                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 848                 neutronPortToDpIdCache.remove(currPortUuid);
 849                 break;
 850             }
 851         }
 852     }
 853
 854     //
 855     // Internal helpers
 856     //
 857     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 858
 859         final String networkUUID = neutronPort.getNetworkUUID();
 860         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 861
 862         if(!isDelete) {
 863             // If there is no router interface handling the networkUUID, we are done
 864             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 865                 return;
 866             }
 867
 868             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 869             // router interface are handled via handleNeutronRouterInterfaceEvent.
 870             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 871                 return;
 872             }
 873         }
 874
 875         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 876         final String providerSegmentationId = neutronNetwork != null ?
 877                                               neutronNetwork.getProviderSegmentationID() : null;
 878         final String tenantMac = neutronPort.getMacAddress();
 879
 880         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 881             tenantMac == null || tenantMac.isEmpty()) {
 882             // done: go no further w/out all the info needed...
 883             return;
 884         }
 885
 886         final Action action = isDelete ? DELETE : ADD;
 887         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 888         if (nodes.isEmpty()) {
 889             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 890         }
 891         for (Node node : nodes) {
 892             final Long dpid = getDpidForIntegrationBridge(node);
 893             if (dpid == null) {
 894                 continue;
 895             }
 896             if (neutronPort.getFixedIPs() == null) {
 897                 continue;
 898             }
 899             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 900                 final String tenantIpStr = neutronIP.getIpAddress();
 901                 if (tenantIpStr.isEmpty()) {
 902                     continue;
 903                 }
 904
 905                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 906                 // still needed when routing to subnets non-local to node (bug 2076).
 907                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 908             }
 909         }
 910     }
 911
 912     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 913         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 914         /**
 915          * Get updated data and original data for the the changed. Identify the security groups that got
 916          * added and removed and call the appropriate providers for updating the flows.
 917          */
 918         try {
 919             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 920                                                                             neutronPort.getOriginalPort());
 921             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 922                                                                               neutronPort);
 923
 924             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 925                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 926             }
 927             if (null != addedGroup && !addedGroup.isEmpty()) {
 928                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 929             }
 930
 931         } catch (Exception e) {
 932             LOG.error("Exception in processSecurityGroupUpdate", e);
 933         }
 934     }
 935
 936     private void processPortSecurityEnableUpdated(NeutronPort neutronPort) {
 937         LOG.trace("processPortSecurityEnableUpdated:" + neutronPort);
 938         securityServicesManager.syncFixedSecurityGroup(neutronPort,
 939             neutronPort.getPortSecurityEnabled());
 940     }
 941
 942     private boolean isPortSecurityEnableUpdated(NeutronPort neutronPort) {
 943         LOG.trace("isPortSecurityEnableUpdated: {}", neutronPort);
 944         if (neutronPort == null) {
 945             return false;
 946         }
 947
 948         NeutronPort originalPort = neutronPort.getOriginalPort();
 949         if (originalPort == null) {
 950             return false;
 951         }
 952
 953         Boolean originalPortSecurityEnabled = originalPort.getPortSecurityEnabled();
 954         if (originalPortSecurityEnabled == null) {
 955             return false;
 956         }
 957
 958         return !originalPortSecurityEnabled.equals(neutronPort.getPortSecurityEnabled());
 959     }
 960
 961     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 962         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 963         if (port1 == null) {
 964             return null;
 965         }
 966         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 967         if (port2 == null) {
 968             return list1;
 969         }
 970         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 971         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 972             NeutronSecurityGroup securityGroup1 = iterator.next();
 973             for (NeutronSecurityGroup securityGroup2 :list2) {
 974                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 975                     iterator.remove();
 976                 }
 977             }
 978         }
 979         return list1;
 980     }
 981
 982     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 983                                            String macAddress, String ipStr,
 984                                            Action actionForNode) {
 985         if (actionForNode == DELETE) {
 986             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 987                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 988         }
 989         if (actionForNode == ADD) {
 990             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 991                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 992         }
 993
 994         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 995                                                        macAddress, ipStr, actionForNode);
 996     }
 997
 998     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 999                                              String macAddress,
1000                                              String address,
1001                                              Action actionForNode) {
1002         Status status;
1003         try {
1004             InetAddress inetAddress = InetAddress.getByName(address);
1005             status = l3ForwardingProvider == null ?
1006                      new Status(StatusCode.SUCCESS) :
1007                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
1008                                                                       inetAddress, macAddress, actionForNode);
1009         } catch (UnknownHostException e) {
1010             status = new Status(StatusCode.BADREQUEST);
1011         }
1012
1013         if (status.isSuccess()) {
1014             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
1015                          l3ForwardingProvider == null ? "skipped" : "programmed",
1016                          macAddress, address, node.getNodeId().getValue(), actionForNode);
1017         } else {
1018             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
1019                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
1020         }
1021         return status;
1022     }
1023
1024     // --
1025
1026     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
1027                                                        Boolean isDelete) {
1028         Preconditions.checkNotNull(destNeutronRouterInterface);
1029
1030         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
1031         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
1032         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
1033         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
1034         final NeutronNetwork neutronNetwork = subnet != null ?
1035                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
1036         final String destinationSegmentationId = neutronNetwork != null ?
1037                                                  neutronNetwork.getProviderSegmentationID() : null;
1038         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
1039         final String cidr = subnet != null ? subnet.getCidr() : null;
1040         final int mask = getMaskLenFromCidr(cidr);
1041
1042         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1043                      destNeutronRouterInterface, isDelete);
1044
1045         if (subnet != null && subnet.getIpVersion() == 6) {
1046             LOG.trace("programFlowsForNeutronRouterInterface doesn't support IPv6 router interface");
1047             return;
1048         }
1049         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1050         // the local cache
1051         if (neutronNetwork != null) {
1052             if (macAddress == null || macAddress.isEmpty()) {
1053                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1054             }
1055             if (ipList == null || ipList.isEmpty()) {
1056                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1057             }
1058         }
1059
1060         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1061             cidr == null || cidr.isEmpty() ||
1062             macAddress == null || macAddress.isEmpty() ||
1063             ipList == null || ipList.isEmpty()) {
1064             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1065                          destinationSegmentationId, cidr, macAddress, ipList);
1066             // done: go no further w/out all the info needed...
1067             return;
1068         }
1069
1070         final Action actionForNode = isDelete ? DELETE : ADD;
1071
1072         // Keep cache for finding router's mac from network uuid -- add
1073         //
1074         if (! isDelete) {
1075             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1076             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1077             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1078         }
1079
1080         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1081         if (nodes.isEmpty()) {
1082             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1083         }
1084         for (Node node : nodes) {
1085             final Long dpid = getDpidForIntegrationBridge(node);
1086             if (dpid == null) {
1087                 continue;
1088             }
1089
1090             for (Neutron_IPs neutronIP : ipList) {
1091                 final String ipStr = neutronIP.getIpAddress();
1092                 if (ipStr.isEmpty()) {
1093                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1094                             node.getNodeId().getValue(), ipStr);
1095                     continue;
1096                 }
1097                 final IpAddress ipAddress = new IpAddress(ipStr.toCharArray());
1098                 if (ipAddress.getIpv4Address() == null) {
1099                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ipv6 {}",
1100                             node.getNodeId().getValue(), ipStr);
1101                     continue;
1102                 }
1103
1104                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1105                 //
1106                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1107                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1108                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1109                                                               neutronNetwork, destinationSegmentationId,
1110                                                               macAddress, ipStr, mask, actionForNode,
1111                                                               true /*isReflexsive*/);
1112                 }
1113
1114                 if (! isExternal) {
1115                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1116                             actionForNode);
1117                 }
1118                 // Enable ARP responder by default, because router interface needs to be responded always.
1119                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1120                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1121             }
1122
1123             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1124             // for the external neutron networks.
1125             //
1126             {
1127                 final Action actionForRewriteExclusion = isExternal ? DELETE : actionForNode;
1128                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1129             }
1130         }
1131
1132         if (isDelete) {
1133             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1134             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1135             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1136         }
1137     }
1138
1139     private void programFlowForNetworkFromExternal(final Node node,
1140                                                    final Long dpid,
1141                                                    final String destinationSegmentationId,
1142                                                    final String dstMacAddress,
1143                                                    final String destIpStr,
1144                                                    final int destMask,
1145                                                    final Action actionForNode) {
1146         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1147                 dstMacAddress, destIpStr, destMask, actionForNode);
1148     }
1149
1150     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1151                                                            final Long dpid,
1152                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1153                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1154                                                            final NeutronNetwork dstNeutronNetwork,
1155                                                            final String destinationSegmentationId,
1156                                                            final String dstMacAddress,
1157                                                            final String destIpStr,
1158                                                            final int destMask,
1159                                                            final Action actionForNode,
1160                                                            Boolean isReflexsive) {
1161         Preconditions.checkNotNull(srcNeutronRouterInterface);
1162         Preconditions.checkNotNull(dstNeutronRouterInterface);
1163
1164         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1165         if (sourceSubnetId == null) {
1166             LOG.error("Could not get provider Subnet ID from router interface {}",
1167                          srcNeutronRouterInterface.getID());
1168             return;
1169         }
1170
1171         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1172         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1173         if (sourceNetworkId == null) {
1174             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1175             return;
1176         }
1177
1178         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1179         if (sourceNetwork == null) {
1180             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1181             return;
1182         }
1183
1184         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1185             // Isolate subnets from different tenants within the same router
1186             return;
1187         }
1188         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1189         if (sourceSegmentationId == null) {
1190             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1191             return;
1192         }
1193         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1194             // Skip 'self'
1195             return;
1196         }
1197
1198         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1199                                      dstMacAddress, destIpStr, destMask, actionForNode);
1200
1201         // Flip roles src->dst; dst->src
1202         if (isReflexsive) {
1203             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1204             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1205             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1206             final String cidr2 = sourceSubnet.getCidr();
1207             final int mask2 = getMaskLenFromCidr(cidr2);
1208
1209             if (cidr2 == null || cidr2.isEmpty() ||
1210                 macAddress2 == null || macAddress2.isEmpty() ||
1211                 ipList2 == null || ipList2.isEmpty()) {
1212                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1213                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1214                 // done: go no further w/out all the info needed...
1215                 return;
1216             }
1217
1218             for (Neutron_IPs neutronIP2 : ipList2) {
1219                 final String ipStr2 = neutronIP2.getIpAddress();
1220                 if (ipStr2.isEmpty()) {
1221                     continue;
1222                 }
1223                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1224                                                           srcNeutronRouterInterface,
1225                                                           sourceNetwork, sourceSegmentationId,
1226                                                           macAddress2, ipStr2, mask2, actionForNode,
1227                                                           false /*isReflexsive*/);
1228             }
1229         }
1230     }
1231
1232     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1233                                               String destinationSegmentationId,
1234                                               String macAddress, String ipStr, int mask,
1235                                               Action actionForNode) {
1236         if (actionForNode == DELETE) {
1237             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1238                          " action {}",
1239                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1240                          macAddress, ipStr, mask, actionForNode);
1241         }
1242         if (actionForNode == ADD) {
1243             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1244                          " action {}",
1245                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1246                          macAddress, ipStr, mask, actionForNode);
1247         }
1248
1249         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1250                                                           macAddress, ipStr, mask, actionForNode);
1251     }
1252
1253     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1254                                                 String destinationSegmentationId,
1255                                                 String macAddress,
1256                                                 String address, int mask,
1257                                                 Action actionForNode) {
1258         Status status;
1259         try {
1260             InetAddress inetAddress = InetAddress.getByName(address);
1261             status = routingProvider == null ?
1262                      new Status(StatusCode.SUCCESS) :
1263                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1264                                                             macAddress, inetAddress, mask, actionForNode);
1265         } catch (UnknownHostException e) {
1266             status = new Status(StatusCode.BADREQUEST);
1267         }
1268
1269         if (status.isSuccess()) {
1270             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1271                          routingProvider == null ? "skipped" : "programmed",
1272                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1273                          actionForNode);
1274         } else {
1275             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1276                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1277                          actionForNode, status);
1278         }
1279         return status;
1280     }
1281
1282     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1283                                            String macAddress, String ipStr,
1284                                            Action action) {
1285         if (action == DELETE ) {
1286             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1287                     dpid, segOrOfPort, macAddress, ipStr, action);
1288         }
1289         if (action == ADD) {
1290             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1291                     dpid, segOrOfPort, macAddress, ipStr, action);
1292         }
1293
1294         Status status = new Status(StatusCode.UNSUPPORTED);
1295         if (icmpEchoProvider != null){
1296             try {
1297                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1298                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1299                                                 macAddress, inetAddress, action);
1300             } catch (UnknownHostException e) {
1301                 status = new Status(StatusCode.BADREQUEST);
1302             }
1303         }
1304
1305         if (status.isSuccess()) {
1306             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1307                     icmpEchoProvider == null ? "skipped" : "programmed",
1308                     macAddress, ipStr, dpid, segOrOfPort, action);
1309         } else {
1310             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1311                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1312         }
1313
1314         return status.isSuccess();
1315     }
1316
1317     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1318                                                   String matchAddress, String rewriteAddress,
1319                                                   Action action) {
1320         if (action == DELETE ) {
1321             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1322                     " action {}",
1323                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1324         }
1325         if (action == ADD ) {
1326             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1327                     " action {}",
1328                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1329         }
1330
1331         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1332                 rewriteAddress, action);
1333         return status.isSuccess();
1334     }
1335
1336     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1337                                                  String matchAddress, String rewriteAddress,
1338                                                  Action action) {
1339         Status status;
1340         try {
1341             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1342             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1343             status = inboundNatProvider == null ?
1344                     new Status(StatusCode.SUCCESS) :
1345                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1346                             inetMatchAddress, inetRewriteAddress,
1347                             action);
1348         } catch (UnknownHostException e) {
1349             status = new Status(StatusCode.BADREQUEST);
1350         }
1351
1352         if (status.isSuccess()) {
1353             final boolean isSkipped = inboundNatProvider == null;
1354             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1355                     isSkipped ? "skipped" : "programmed",
1356                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1357         } else {
1358             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1359                          " status:{}",
1360                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1361                     status);
1362         }
1363         return status;
1364     }
1365
1366     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1367                                                  Action actionForRewriteExclusion) {
1368         if (actionForRewriteExclusion == DELETE ) {
1369             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1370                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1371         }
1372         if (actionForRewriteExclusion == ADD) {
1373             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1374                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1375         }
1376
1377         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1378     }
1379
1380     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1381                                                    Action actionForNode) {
1382         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1383                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1384
1385         if (status.isSuccess()) {
1386             final boolean isSkipped = outboundNatProvider == null;
1387             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1388                          isSkipped ? "skipped" : "programmed",
1389                          cidr, node.getNodeId().getValue(), actionForNode);
1390         } else {
1391             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1392                          cidr, node.getNodeId().getValue(), actionForNode, status);
1393         }
1394         return status;
1395     }
1396
1397     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1398
1399         if (action == DELETE) {
1400             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1401                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1402         }
1403         if (action == ADD) {
1404             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1405                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1406         }
1407
1408         this.programOutboundIpRewriteStage2(fid, action);
1409     }
1410
1411     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1412         Status status;
1413         try {
1414             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1415             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1416             status = outboundNatProvider == null ?
1417                     new Status(StatusCode.SUCCESS) :
1418                     outboundNatProvider.programIpRewriteRule(
1419                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1420                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1421         } catch (UnknownHostException e) {
1422             status = new Status(StatusCode.BADREQUEST);
1423         }
1424
1425         if (status.isSuccess()) {
1426             final boolean isSkipped = outboundNatProvider == null;
1427             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1428                             " action {}",
1429                          isSkipped ? "skipped" : "programmed",
1430                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1431         } else {
1432             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1433                          " action {} status:{}",
1434                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1435         }
1436         return status;
1437     }
1438
1439     private int getMaskLenFromCidr(String cidr) {
1440         if (cidr == null) {
1441             return 0;
1442         }
1443         String[] splits = cidr.split("/");
1444         if (splits.length != 2) {
1445             return 0;
1446         }
1447
1448         int result;
1449         try {
1450             result = Integer.parseInt(splits[1].trim());
1451         } catch (NumberFormatException nfe) {
1452             result = 0;
1453         }
1454         return result;
1455     }
1456
1457     private Long getDpidForIntegrationBridge(Node node) {
1458         // Check if node is integration bridge; and only then return its dpid
1459         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1460             return southbound.getDataPathId(node);
1461         }
1462         return null;
1463     }
1464
1465     private Long getDpidForExternalBridge(Node node) {
1466         // Check if node is external bridge; and only then return its dpid
1467         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1468             return southbound.getDataPathId(node);
1469         }
1470         return null;
1471     }
1472
1473     private Node getExternalBridgeNode(){
1474         //Pickup the first node that has external bridge (br-ex).
1475         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1476         //the external network is reachable from every br-ex
1477         // TODO: Consider other deployment scenario, and thing of better solution.
1478         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1479         for(Node node : allBridges){
1480             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1481                 return node;
1482             }
1483         }
1484         return null;
1485     }
1486
1487     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1488         if (gatewayPort.getFixedIPs() == null) {
1489             return null;
1490         }
1491         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1492             String subnetUUID = neutronIPs.getSubnetUUID();
1493             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1494             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1495                 return extSubnet;
1496             }
1497             if (extSubnet == null) {
1498                 // TODO: when subnet is created, try again.
1499                 LOG.debug("subnet {} in not found", subnetUUID);
1500              }
1501         }
1502         return null;
1503     }
1504
1505     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1506         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1507         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1508             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1509                 // Neutron floating Ip's port uuid cannot be null (Bug#5894)
1510                 if (neutronFloatingIP.getPortUUID() != null &&
1511                         (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID()))) {
1512                     handleNeutronFloatingIPEvent(neutronFloatingIP, DELETE);
1513                 }
1514             }
1515         }
1516     }
1517
1518     private void updateFloatingIPRules(final NeutronPort neutronPort) {
1519         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1520         if (neutronFloatingIps != null) {
1521             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1522                 if (neutronFloatingIP.getPortUUID() != null
1523                         && neutronFloatingIP.getPortUUID().equals(
1524                                 neutronPort.getPortUUID())) {
1525                     handleNeutronFloatingIPEvent(neutronFloatingIP, UPDATE);
1526                 }
1527             }
1528         }
1529     }
1530
1531
1532     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1533
1534         Preconditions.checkNotNull(gatewayPort);
1535         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1536
1537         if(externalNetwork != null){
1538             if(externalNetwork.isRouterExternal()){
1539                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1540
1541                 // TODO: address IPv6 case.
1542                 if (externalSubnet != null &&
1543                         externalSubnet.getIpVersion() == 4 &&
1544                         gatewayPort.getFixedIPs() != null) {
1545                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1546                     Neutron_IPs neutronIP = null;
1547                     for (Neutron_IPs nIP : gatewayPort.getFixedIPs()) {
1548                         InetAddress ipAddress;
1549                         try {
1550                             ipAddress = InetAddress.getByName(nIP.getIpAddress());
1551                         } catch (UnknownHostException e) {
1552                             LOG.warn("unknown host exception {}", e);
1553                             continue;
1554                         }
1555                         if (ipAddress instanceof Inet4Address) {
1556                             neutronIP = nIP;
1557                             break;
1558                         }
1559                     }
1560                     if (neutronIP == null) {
1561                         // TODO IPv6 neighbor discovery
1562                         LOG.debug("Ignoring gateway ports with IPv6 only fixed ip {}",
1563                                   gatewayPort.getFixedIPs());
1564                     } else {
1565                         gatewayMacResolver.resolveMacAddress(
1566                             this, /* gatewayMacResolverListener */
1567                             null, /* externalNetworkBridgeDpid */
1568                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1569                             new Ipv4Address(externalSubnet.getGatewayIP()),
1570                             new Ipv4Address(neutronIP.getIpAddress()),
1571                             new MacAddress(gatewayPort.getMacAddress()),
1572                             true /* periodicRefresh */);
1573                     }
1574                 } else {
1575                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1576                 }
1577             }
1578         }else{
1579             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1580         }
1581     }
1582
1583
1584     private void storePortInCleanupCache(NeutronPort port) {
1585         this.portCleanupCache.put(port.getPortUUID(),port);
1586     }
1587
1588
1589     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1590         removePortFromCleanupCache(originalPort);
1591         storePortInCleanupCache(updatedPort);
1592     }
1593
1594     public void removePortFromCleanupCache(NeutronPort port) {
1595         if(port != null) {
1596             this.portCleanupCache.remove(port.getPortUUID());
1597         }
1598     }
1599
1600     public Map<String, NeutronPort> getPortCleanupCache() {
1601         return this.portCleanupCache;
1602     }
1603
1604     public NeutronPort getPortFromCleanupCache(String portid) {
1605         NeutronPort ret = this.portCleanupCache.get(portid);
1606         if (ret != null) {
1607             LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1608         }
1609         return ret;
1610     }
1611
1612     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1613         if (network != null) {
1614             this.networkCleanupCache.put(network.getNetworkUUID(), network);
1615         }
1616     }
1617
1618
1619     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1620         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1621     }
1622
1623     public void removeNetworkFromCleanupCache(String networkid) {
1624         NeutronNetwork network = networkCleanupCache.get(networkid);
1625         if (network != null) {
1626             for (NeutronPort port : this.portCleanupCache.values()) {
1627                 if (port.getNetworkUUID().equals(networkid)) {
1628                     LOG.info("This network is used by another port", network);
1629                     return;
1630                 }
1631             }
1632             this.networkCleanupCache.remove(network.getNetworkUUID());
1633         }
1634     }
1635
1636     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1637         return this.networkCleanupCache;
1638     }
1639
1640     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1641         NeutronNetwork ret = networkCleanupCache.get(networkid);
1642         if (ret != null) {
1643             LOG.info("getNetworkFromCleanupCache: Matching NeutronNetwork found {}", networkid);
1644         }
1645         return ret;
1646     }
1647     /**
1648      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1649      *
1650      * @param ofPort the OF port number
1651      * @return the string with encoded OF port (example format "OFPort|999")
1652      */
1653     public static String encodeExcplicitOFPort(Long ofPort) {
1654         return "OFPort|" + ofPort.toString();
1655     }
1656     private void initNetworkCleanUpCache() {
1657         if (this.neutronNetworkCache != null) {
1658             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1659                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1660             }
1661         }
1662     }
1663     private void initPortCleanUpCache() {
1664         if (this.neutronPortCache != null) {
1665             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1666                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1667             }
1668         }
1669     }
1670     @Override
1671     public void setDependencies(ServiceReference serviceReference) {
1672         eventDispatcher =
1673                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1674         eventDispatcher.eventHandlerAdded(serviceReference, this);
1675         tenantNetworkManager =
1676                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1677         configurationService =
1678                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1679         arpProvider =
1680                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1681         inboundNatProvider =
1682                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1683         outboundNatProvider =
1684                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1685         routingProvider =
1686                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1687         l3ForwardingProvider =
1688                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1689         distributedArpService =
1690                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1691         nodeCacheManager =
1692                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1693         southbound =
1694                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1695         gatewayMacResolver =
1696                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1697         securityServicesManager =
1698                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1699
1700         initL3AdapterMembers();
1701     }
1702
1703     @Override
1704     public void setDependencies(Object impl) {
1705         if (impl instanceof INeutronNetworkCRUD) {
1706             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1707             initNetworkCleanUpCache();
1708         } else if (impl instanceof INeutronPortCRUD) {
1709             neutronPortCache = (INeutronPortCRUD)impl;
1710             initPortCleanUpCache();
1711         } else if (impl instanceof INeutronSubnetCRUD) {
1712             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1713         } else if (impl instanceof INeutronFloatingIPCRUD) {
1714             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1715         } else if (impl instanceof ArpProvider) {
1716             arpProvider = (ArpProvider)impl;
1717         } else if (impl instanceof InboundNatProvider) {
1718             inboundNatProvider = (InboundNatProvider)impl;
1719         } else if (impl instanceof OutboundNatProvider) {
1720             outboundNatProvider = (OutboundNatProvider)impl;
1721         } else if (impl instanceof RoutingProvider) {
1722             routingProvider = (RoutingProvider)impl;
1723         } else if (impl instanceof L3ForwardingProvider) {
1724             l3ForwardingProvider = (L3ForwardingProvider)impl;
1725         }else if (impl instanceof GatewayMacResolver) {
1726             gatewayMacResolver = (GatewayMacResolver)impl;
1727         }else if (impl instanceof IcmpEchoProvider) {
1728             icmpEchoProvider = (IcmpEchoProvider)impl;
1729         }
1730
1731         populateL3ForwardingCaches();
1732     }
1733 }