openstack/net-virt/src/main/java/org/opendaylight/netvirt/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright © 2014, 2017 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.openstack.netvirt.impl;
  10
  11 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.ADD;
  12 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.DELETE;
  13 import static org.opendaylight.netvirt.openstack.netvirt.api.Action.UPDATE;
  14
  15 import com.google.common.base.Preconditions;
  16 import org.apache.commons.lang3.tuple.ImmutablePair;
  17 import org.apache.commons.lang3.tuple.Pair;
  18 import org.opendaylight.netvirt.openstack.netvirt.AbstractEvent;
  19 import org.opendaylight.netvirt.openstack.netvirt.AbstractHandler;
  20 import org.opendaylight.netvirt.openstack.netvirt.ConfigInterface;
  21 import org.opendaylight.netvirt.openstack.netvirt.NeutronL3AdapterEvent;
  22 import org.opendaylight.netvirt.openstack.netvirt.NeutronModelsDataStoreHelper;
  23 import org.opendaylight.netvirt.openstack.netvirt.api.Action;
  24 import org.opendaylight.netvirt.openstack.netvirt.api.ArpProvider;
  25 import org.opendaylight.netvirt.openstack.netvirt.api.ConfigurationService;
  26 import org.opendaylight.netvirt.openstack.netvirt.api.Constants;
  27 import org.opendaylight.netvirt.openstack.netvirt.api.EventDispatcher;
  28 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolver;
  29 import org.opendaylight.netvirt.openstack.netvirt.api.GatewayMacResolverListener;
  30 import org.opendaylight.netvirt.openstack.netvirt.api.IcmpEchoProvider;
  31 import org.opendaylight.netvirt.openstack.netvirt.api.InboundNatProvider;
  32 import org.opendaylight.netvirt.openstack.netvirt.api.L3ForwardingProvider;
  33 import org.opendaylight.netvirt.openstack.netvirt.api.NodeCacheManager;
  34 import org.opendaylight.netvirt.openstack.netvirt.api.OutboundNatProvider;
  35 import org.opendaylight.netvirt.openstack.netvirt.api.RoutingProvider;
  36 import org.opendaylight.netvirt.openstack.netvirt.api.SecurityServicesManager;
  37 import org.opendaylight.netvirt.openstack.netvirt.api.Southbound;
  38 import org.opendaylight.netvirt.openstack.netvirt.api.Status;
  39 import org.opendaylight.netvirt.openstack.netvirt.api.StatusCode;
  40 import org.opendaylight.netvirt.openstack.netvirt.api.TenantNetworkManager;
  41 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronFloatingIP;
  42 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronNetwork;
  43 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter;
  44 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronRouter_Interface;
  45 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSecurityGroup;
  46 import org.opendaylight.netvirt.openstack.netvirt.translator.Neutron_IPs;
  47 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  48 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  49 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronPortCRUD;
  50 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronPort;
  51 import org.opendaylight.netvirt.openstack.netvirt.translator.NeutronSubnet;
  52 import org.opendaylight.netvirt.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  53 import org.opendaylight.netvirt.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  54 import org.opendaylight.netvirt.utils.servicehelper.ServiceHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4Address;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.port.attributes.FixedIps;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  65 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  66 import org.osgi.framework.ServiceReference;
  67 import org.slf4j.Logger;
  68 import org.slf4j.LoggerFactory;
  69
  70 import java.net.Inet4Address;
  71 import java.net.InetAddress;
  72 import java.net.UnknownHostException;
  73 import java.util.ArrayList;
  74 import java.util.Collection;
  75 import java.util.HashMap;
  76 import java.util.Iterator;
  77 import java.util.List;
  78 import java.util.Map;
  79 import java.util.concurrent.ConcurrentHashMap;
  80 import java.util.concurrent.ConcurrentMap;
  81
  82 /**
  83  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  84  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  85  * as well as the multi-tenant router forwarding provider.
  86  */
  87 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  88     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  89
  90     // The implementation for each of these services is resolved by the OSGi Service Manager
  91     private volatile ConfigurationService configurationService;
  92     private volatile TenantNetworkManager tenantNetworkManager;
  93     private volatile NodeCacheManager nodeCacheManager;
  94     private volatile INeutronNetworkCRUD neutronNetworkCache;
  95     private volatile INeutronSubnetCRUD neutronSubnetCache;
  96     private volatile INeutronPortCRUD neutronPortCache;
  97     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  98     private volatile L3ForwardingProvider l3ForwardingProvider;
  99     private volatile InboundNatProvider inboundNatProvider;
 100     private volatile OutboundNatProvider outboundNatProvider;
 101     private volatile ArpProvider arpProvider;
 102     private volatile RoutingProvider routingProvider;
 103     private volatile GatewayMacResolver gatewayMacResolver;
 104     private volatile SecurityServicesManager securityServicesManager;
 105     private volatile IcmpEchoProvider icmpEchoProvider;
 106
 107     private class FloatIpData {
 108         // br-int of node where floating ip is associated with tenant port
 109         private final Long dpid;
 110         // patch port in br-int used to reach br-ex
 111         private final Long ofPort;
 112         // segmentation id of the net where fixed ip is instantiated
 113         private final String segId;
 114         // mac address assigned to neutron port of floating ip
 115         private final String macAddress;
 116         private final String floatingIpAddress;
 117         // ip address given to tenant vm
 118         private final String fixedIpAddress;
 119         private final String neutronRouterMac;
 120
 121         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 122                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 123             this.dpid = dpid;
 124             this.ofPort = ofPort;
 125             this.segId = segId;
 126             this.macAddress = macAddress;
 127             this.floatingIpAddress = floatingIpAddress;
 128             this.fixedIpAddress = fixedIpAddress;
 129             this.neutronRouterMac = neutronRouterMac;
 130         }
 131     }
 132
 133     private Map<String, String> networkIdToRouterMacCache;
 134     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 135     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 136
 137     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 138     private Map<String, FloatIpData> floatIpDataMapCache;
 139
 140     private String externalRouterMac;
 141     private Boolean enabled = false;
 142     private Boolean isCachePopulationDone = false;
 143     private ConcurrentMap<String, NeutronPort> portCleanupCache;
 144     private ConcurrentMap<String, NeutronNetwork> networkCleanupCache;
 145
 146     private Southbound southbound;
 147     private DistributedArpService distributedArpService;
 148     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 149
 150     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 151     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 152     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 153     private static final String OWNER_FLOATING_IP = "network:floatingip";
 154     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 155
 156     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 157         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 158         this.neutronModelsDataStoreHelper = neutronHelper;
 159     }
 160
 161     private void initL3AdapterMembers() {
 162         Preconditions.checkNotNull(configurationService);
 163
 164         if (configurationService.isL3ForwardingEnabled()) {
 165             this.networkIdToRouterMacCache = new HashMap<>();
 166             this.networkIdToRouterIpListCache = new HashMap<>();
 167             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 168             this.floatIpDataMapCache = new HashMap<>();
 169
 170             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 171             if (this.externalRouterMac == null) {
 172                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 173             }
 174             this.enabled = true;
 175             LOG.info("OVSDB L3 forwarding is enabled");
 176         } else {
 177             LOG.debug("OVSDB L3 forwarding is disabled");
 178         }
 179         this.neutronPortToDpIdCache = new HashMap<>();
 180         this.portCleanupCache = new ConcurrentHashMap<>();
 181         this.networkCleanupCache = new ConcurrentHashMap<>();
 182     }
 183
 184     //
 185     // Callbacks from AbstractHandler
 186     //
 187     @Override
 188     public void processEvent(AbstractEvent abstractEvent) {
 189         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 190             LOG.error("Unable to process abstract event " + abstractEvent);
 191             return;
 192         }
 193         if (!this.enabled) {
 194             return;
 195         }
 196
 197         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 198         switch (ev.getAction()) {
 199             case UPDATE:
 200                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 201                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 202                 } else {
 203                     LOG.warn("Received update for an unexpected event " + ev);
 204                 }
 205                 break;
 206             case ADD:
 207                 // fall through...
 208                 // break;
 209             case DELETE:
 210                 // fall through...
 211                 // break;
 212             default:
 213                 LOG.warn("Unable to process event " + ev);
 214                 break;
 215         }
 216     }
 217
 218     //
 219     // Callbacks from GatewayMacResolverListener
 220     //
 221
 222     @Override
 223     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 224         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 225                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 226         if (!this.enabled) {
 227             return;
 228         }
 229
 230         if (macAddress == null || macAddress.getValue() == null) {
 231             // TODO: handle cases when mac is null
 232             return;
 233         }
 234
 235         //
 236         // Enqueue event so update is handled by adapter's thread
 237         //
 238         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 239     }
 240
 241     private void populateL3ForwardingCaches() {
 242         if (!this.enabled) {
 243             return;
 244         }
 245         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 246                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 247             return;
 248         }
 249         this.isCachePopulationDone = true;
 250         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 251         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 252         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 253         if(routers != null && routers.getRouter() != null && ports != null) {
 254             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 255             for( Router router : routers.getRouter()) {
 256                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 257                 if(!ports.getPort().isEmpty()) {
 258                     for( Port port : ports.getPort()) {
 259                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 260                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 261                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 262                             for (final FixedIps fixedIps : port.getFixedIps()) {
 263                                 if (fixedIps.getIpAddress().getIpv4Address() != null) {
 264                                     networkIdToRouterMacCache.put(port.getNetworkId().getValue(),
 265                                                                   port.getMacAddress().getValue());
 266                                     networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 267                                                                      NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 268                                     subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 269                                                                        NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 270                                     break;
 271                                 }
 272                             }
 273                         }
 274                     }
 275                 }else {
 276                     LOG.warn("L3 Cache Population :Did not find any port information " +
 277                             "in config Data Store for router {}",router);
 278                 }
 279             }
 280         }
 281         LOG.debug("NetVirt L3 caches population is done");
 282     }
 283
 284     public Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 285         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 286             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 287             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 288             for(Node bridge : bridges) {
 289                 List<OvsdbTerminationPointAugmentation> interfaces
 290                         = southbound.extractTerminationPointAugmentations(bridge);
 291                 if(interfaces != null && !interfaces.isEmpty()) {
 292                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 293                             interfaces.size(), bridge.getNodeId());
 294                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 295                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 296                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 297                             Long dpId = getDpidForIntegrationBridge(bridge);
 298                             Uuid interfaceUuid = intf.getInterfaceUuid();
 299                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 300                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 301                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 302                             break;
 303                         }
 304                     }
 305                 }
 306             }
 307         }
 308         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 309     }
 310
 311     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 312         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 313         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 314         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 315             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 316                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 317                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 318                             "Fetching from data store.",neutronFloatingIP.getID());
 319                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 320                 }
 321             }
 322         }
 323         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 324         return floatIpDataMapCache.values();
 325     }
 326     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 327         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 328                 "floating id {} ",neutronFloatingId);
 329         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 330             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 331             if (neutronFloatingIP == null) {
 332                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 333                 return null;
 334             }
 335             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 336             NeutronPort neutronPortForFloatIp = null;
 337             for (NeutronPort neutronPort : neutronPorts) {
 338                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 339                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 340                     neutronPortForFloatIp = neutronPort;
 341                     break;
 342                 }
 343             }
 344
 345             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 346             if(neutronTenantPortUuid == null) {
 347                 return null;
 348             }
 349             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 350             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 351             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 352             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 353
 354             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 355             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 356                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 357             String providerSegmentationId = tenantNeutronNetwork != null ?
 358                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 359             String neutronRouterMac = tenantNeutronNetwork != null ?
 360                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 361
 362             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 363                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 364                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 365                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 366                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 367                                 "seg {} mac {} rtrMac {}",
 368                         fixedIpAddress,
 369                         floatingIpAddress,
 370                         neutronPortForFloatIp,
 371                         neutronTenantPortUuid,
 372                         providerSegmentationId,
 373                         floatingIpMac,
 374                         neutronRouterMac);
 375
 376                 return null;
 377             }
 378
 379             // get ofport for patch port in br-int
 380             final Long dpId = nodeIfPair.getLeft();
 381             final Long ofPort = findOFPortForExtPatch(dpId);
 382             if (ofPort == null) {
 383                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 384                                 "to connect floating ip to external bridge. dpid {}",
 385                         dpId);
 386                 return null;
 387             }
 388
 389             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 390                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 391             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 392
 393         }
 394         return floatIpDataMapCache.get(neutronFloatingId);
 395     }
 396     /**
 397      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 398      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 399      * subscribe so that the mac address associated to the gateway ip address is available.
 400      *
 401      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 402      */
 403     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 404         Preconditions.checkNotNull(externalRouterMacUpdate);
 405
 406         flushExistingIpRewrite();
 407         this.externalRouterMac = externalRouterMacUpdate;
 408         rebuildExistingIpRewrite();
 409     }
 410
 411     /**
 412      * Process the event.
 413      *
 414      * @param action the {@link Action} action to be handled.
 415      * @param subnet An instance of NeutronSubnet object.
 416      */
 417     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 418         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 419         if (action == ADD) {
 420             this.storeNetworkInCleanupCache(neutronNetworkCache.getNetwork(subnet.getNetworkUUID()));
 421         }
 422     }
 423
 424     /**
 425      * Process the port event as a router interface event.
 426      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 427      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 428      *
 429      * @param action the {@link Action} action to be handled.
 430      * @param neutronPort An instance of NeutronPort object.
 431      */
 432     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 433         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 434
 435         if (action == UPDATE) {
 436             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 437             this.updatePortInCleanupCache(neutronPort, neutronPort.getOriginalPort());
 438             if (neutronPort.getPortSecurityEnabled()) {
 439                 this.processSecurityGroupUpdate(neutronPort);
 440             }
 441             if (isPortSecurityEnableUpdated(neutronPort)) {
 442                 this.processPortSecurityEnableUpdated(neutronPort);
 443             }
 444         }
 445
 446         if (!this.enabled) {
 447             return;
 448         }
 449
 450         final boolean isDelete = action == DELETE;
 451
 452         if (action == DELETE) {
 453             // Bug 5164: Cleanup Floating IP OpenFlow Rules when port is deleted.
 454             this.cleanupFloatingIPRules(neutronPort);
 455         }
 456         else if (action == UPDATE){
 457             // Bug 5353: VM restart cause floatingIp flows to be removed
 458             this.updateFloatingIPRules(neutronPort);
 459         }
 460
 461         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 462             if (!isDelete) {
 463                 LOG.info("Port {} is network router gateway interface, "
 464                         + "triggering gateway resolution for the attached external network", neutronPort);
 465                 this.triggerGatewayMacResolver(neutronPort);
 466             }else{
 467                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 468                 if (null == externalNetwork) {
 469                     externalNetwork = this.getNetworkFromCleanupCache(neutronPort.getNetworkUUID());
 470                 }
 471
 472                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 473                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 474                     // TODO support IPv6
 475                     if (externalSubnet != null &&
 476                             externalSubnet.getIpVersion() == 4) {
 477                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 478                     }
 479                 }
 480             }
 481         }
 482
 483         // Treat the port event as a router interface event if the port belongs to router. This is a
 484         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 485         //
 486         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 487             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 488
 489             if (neutronPort.getFixedIPs() != null) {
 490                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 491                     NeutronRouter_Interface neutronRouterInterface =
 492                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 493                     // id of router interface to be same as subnet
 494                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 495                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 496
 497                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 498                 }
 499             }
 500         } else {
 501             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 502             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 503             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 504             // there.
 505             //
 506             if (!isDelete && neutronPort.getFixedIPs() != null) {
 507                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 508                     NeutronRouter_Interface neutronRouterInterface =
 509                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 510                     if (neutronRouterInterface != null) {
 511                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 512                     }
 513                 }
 514             }
 515             this.updateL3ForNeutronPort(neutronPort, isDelete);
 516         }
 517     }
 518
 519     /**
 520      * Process the event.
 521      *
 522      * @param action the {@link Action} action to be handled.
 523      * @param neutronRouter An instance of NeutronRouter object.
 524      */
 525     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 526         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 527     }
 528
 529     /**
 530      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 531      * delete the ports on the same subnet.
 532      *
 533      * @param action the {@link Action} action to be handled.
 534      * @param neutronRouter An instance of NeutronRouter object.
 535      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 536      */
 537     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 538                                                   final NeutronRouter_Interface neutronRouterInterface,
 539                                                   Action action) {
 540         LOG.debug("Router interface {} got event {}. Subnet {}",
 541                      neutronRouterInterface.getPortUUID(),
 542                      action,
 543                      neutronRouterInterface.getSubnetUUID());
 544         if (!this.enabled) {
 545             return;
 546         }
 547
 548         final boolean isDelete = action == DELETE;
 549
 550         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 551
 552         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 553         // see if they are affected by l3
 554         //
 555         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 556             boolean currPortShouldBeDeleted = false;
 557             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 558             if (isDelete) {
 559                 if (neutronPort.getFixedIPs() != null) {
 560                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 561                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 562                             currPortShouldBeDeleted = true;
 563                             break;
 564                         }
 565                     }
 566                 }
 567             }
 568             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 569         }
 570
 571     }
 572
 573     /**
 574      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 575      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 576      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 577      * way around) on OpenFlow Table 100.
 578      *
 579      * @param actionIn the {@link Action} action to be handled.
 580      * @param neutronFloatingIP An {@link NeutronFloatingIP} instance of NeutronFloatingIP object.
 581      */
 582     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 583                                              Action actionIn) {
 584         Preconditions.checkNotNull(neutronFloatingIP);
 585
 586         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 587                 neutronFloatingIP.getFixedIPAddress(),
 588                 neutronFloatingIP.getFloatingIPAddress(),
 589                 neutronFloatingIP.getFloatingNetworkUUID());
 590         if (!this.enabled) {
 591             return;
 592         }
 593
 594         Action action;
 595
 596         // Consider action to be delete if getFixedIPAddress is null
 597         //
 598         if (neutronFloatingIP.getFixedIPAddress() == null) {
 599             action = DELETE;
 600         } else {
 601             action = actionIn;
 602         }
 603
 604         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 605
 606         if (action != DELETE) {
 607             // must be first, as it updates floatIpDataMapCache
 608             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 609
 610             programFlowsForFloatingIPInbound(neutronFloatingIP, ADD);
 611             programFlowsForFloatingIPOutbound(neutronFloatingIP, ADD);
 612         } else {
 613             programFlowsForFloatingIPOutbound(neutronFloatingIP, DELETE);
 614             programFlowsForFloatingIPInbound(neutronFloatingIP, DELETE);
 615
 616             // must be last, as it updates floatIpDataMapCache
 617             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 618         }
 619     }
 620
 621     /**
 622      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 623      * ip, otherwise for newer one.
 624      */
 625     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 626         Preconditions.checkNotNull(neutronFloatingIP);
 627
 628         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 629         if (fid == null) {
 630             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 631                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 632             return;
 633         }
 634         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 635                                       action);
 636     }
 637
 638     /**
 639      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 640      * ip, otherwise for newer one.
 641      */
 642     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 643         Preconditions.checkNotNull(neutronFloatingIP);
 644
 645         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 646         if (fid == null) {
 647             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 648                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 649             return;
 650         }
 651         programOutboundIpRewriteStage1(fid, action);
 652     }
 653
 654     private void flushExistingIpRewrite() {
 655         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 656             programOutboundIpRewriteStage1(fid, DELETE);
 657         }
 658     }
 659
 660     private void rebuildExistingIpRewrite() {
 661         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 662             programOutboundIpRewriteStage1(fid, ADD);
 663         }
 664     }
 665
 666     /**
 667      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 668      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 669      * requests.
 670      */
 671     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 672         Preconditions.checkNotNull(neutronFloatingIP);
 673         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 674         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 675
 676         // find bridge Node where floating ip is configured by looking up cache for its port
 677         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 678         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 679         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 680         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 681         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 682         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 683
 684         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 685         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 686                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 687         final String providerSegmentationId = tenantNeutronNetwork != null ?
 688                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 689         final String neutronRouterMac = tenantNeutronNetwork != null ?
 690                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 691
 692         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 693                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 694                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 695                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 696             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 697                     fixedIpAddress,
 698                     floatingIpAddress,
 699                     neutronPortForFloatIp,
 700                     neutronTenantPortUuid,
 701                     providerSegmentationId,
 702                     floatingIpMac,
 703                     neutronRouterMac);
 704             return;
 705         }
 706
 707         // get ofport for patch port in br-int
 708         final Long dpId = nodeIfPair.getLeft();
 709         final Long ofPort = findOFPortForExtPatch(dpId);
 710         if (ofPort == null) {
 711             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 712                     dpId);
 713             return;
 714         }
 715
 716         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 717         //
 718         if (distributedArpService.programStaticRuleStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 719                 ADD)) {
 720             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 721                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 722             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 723             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 724                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 725                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 726         }
 727     }
 728
 729     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 730         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 731         if (floatIpData == null) {
 732             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 733             return;
 734         }
 735
 736         if (distributedArpService.programStaticRuleStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 737                 floatIpData.floatingIpAddress, DELETE)) {
 738             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 739             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 740                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 741         }
 742     }
 743
 744     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 745         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 746             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 747                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 748                 return neutronPort;
 749             }
 750         }
 751         return null;
 752     }
 753
 754     private Long findOFPortForExtPatch(Long dpId) {
 755         final String brInt = configurationService.getIntegrationBridgeName();
 756         final String brExt = configurationService.getExternalBridgeName();
 757         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 758
 759         Preconditions.checkNotNull(dpId);
 760         Preconditions.checkNotNull(portNameInt);
 761
 762         final long dpidPrimitive = dpId;
 763         for (Node node : nodeCacheManager.getBridgeNodes()) {
 764             if (dpidPrimitive == southbound.getDataPathId(node)) {
 765                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 766                         southbound.getTerminationPointOfBridge(node, portNameInt);
 767                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 768             }
 769         }
 770         return null;
 771     }
 772
 773     /**
 774      * Process the event.
 775      *
 776      * @param action the {@link Action} action to be handled.
 777      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronFloatingIP object.
 778      */
 779     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 780         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 781         if (action == UPDATE) {
 782             this.updateNetworkInCleanupCache(neutronNetwork);
 783         }
 784     }
 785
 786     //
 787     // Callbacks from OVSDB's southbound handler
 788     //
 789     /**
 790      * Process the event.
 791      *
 792      * @param bridgeNode An instance of Node object.
 793      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 794      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 795      * @param neutronNetwork An {@link NeutronNetwork} instance of NeutronNetwork
 796      * object.
 797      * @param action the {@link Action} action to be handled.
 798      */
 799     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 800                                      final NeutronNetwork neutronNetwork, Action action) {
 801         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 802                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 803
 804         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 805         if (action != DELETE && neutronPort != null) {
 806             // FIXME: Bug 4971 Move cleanup cache to SG Impl
 807             storePortInCleanupCache(neutronPort);
 808         }
 809
 810         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 811         final Uuid interfaceUuid = intf.getInterfaceUuid();
 812
 813         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 814                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 815
 816         if (neutronPort != null) {
 817             final String neutronPortUuid = neutronPort.getPortUUID();
 818
 819             if (action == ADD && dpId != null && interfaceUuid != null) {
 820                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 821             }
 822
 823             if (this.enabled) {
 824                 handleNeutronPortEvent(neutronPort, action == DELETE ? Action.DELETE : Action.UPDATE);
 825             }
 826         }
 827
 828         if (action == DELETE && interfaceUuid != null) {
 829             handleInterfaceEventDelete(intf, dpId);
 830         }
 831     }
 832
 833     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 834         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 835         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 836                 neutronPortUuid, dpId, interfaceUuid.getValue());
 837     }
 838
 839     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 840         // Remove entry from neutronPortToDpIdCache based on interface uuid
 841         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 842             final String currPortUuid = entry.getKey();
 843             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 844                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 845                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 846                 neutronPortToDpIdCache.remove(currPortUuid);
 847                 break;
 848             }
 849         }
 850     }
 851
 852     //
 853     // Internal helpers
 854     //
 855     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 856
 857         final String networkUUID = neutronPort.getNetworkUUID();
 858         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 859
 860         if(!isDelete) {
 861             // If there is no router interface handling the networkUUID, we are done
 862             if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 863                 return;
 864             }
 865
 866             // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 867             // router interface are handled via handleNeutronRouterInterfaceEvent.
 868             if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 869                 return;
 870             }
 871         }
 872
 873         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 874         final String providerSegmentationId = neutronNetwork != null ?
 875                                               neutronNetwork.getProviderSegmentationID() : null;
 876         final String tenantMac = neutronPort.getMacAddress();
 877
 878         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 879             tenantMac == null || tenantMac.isEmpty()) {
 880             // done: go no further w/out all the info needed...
 881             return;
 882         }
 883
 884         final Action action = isDelete ? DELETE : ADD;
 885         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 886         if (nodes.isEmpty()) {
 887             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 888         }
 889         for (Node node : nodes) {
 890             final Long dpid = getDpidForIntegrationBridge(node);
 891             if (dpid == null) {
 892                 continue;
 893             }
 894             if (neutronPort.getFixedIPs() == null) {
 895                 continue;
 896             }
 897             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 898                 final String tenantIpStr = neutronIP.getIpAddress();
 899                 if (tenantIpStr.isEmpty()) {
 900                     continue;
 901                 }
 902
 903                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 904                 // still needed when routing to subnets non-local to node (bug 2076).
 905                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 906             }
 907         }
 908     }
 909
 910     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 911         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 912         /**
 913          * Get updated data and original data for the the changed. Identify the security groups that got
 914          * added and removed and call the appropriate providers for updating the flows.
 915          */
 916         try {
 917             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 918                                                                             neutronPort.getOriginalPort());
 919             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 920                                                                               neutronPort);
 921
 922             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 923                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 924             }
 925             if (null != addedGroup && !addedGroup.isEmpty()) {
 926                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 927             }
 928
 929         } catch (Exception e) {
 930             LOG.error("Exception in processSecurityGroupUpdate", e);
 931         }
 932     }
 933
 934     private void processPortSecurityEnableUpdated(NeutronPort neutronPort) {
 935         LOG.trace("processPortSecurityEnableUpdated:" + neutronPort);
 936         securityServicesManager.syncFixedSecurityGroup(neutronPort,
 937             neutronPort.getPortSecurityEnabled());
 938     }
 939
 940     private boolean isPortSecurityEnableUpdated(NeutronPort neutronPort) {
 941         LOG.trace("isPortSecurityEnableUpdated: {}", neutronPort);
 942         if (neutronPort == null) {
 943             return false;
 944         }
 945
 946         NeutronPort originalPort = neutronPort.getOriginalPort();
 947         if (originalPort == null) {
 948             return false;
 949         }
 950
 951         Boolean originalPortSecurityEnabled = originalPort.getPortSecurityEnabled();
 952         if (originalPortSecurityEnabled == null) {
 953             return false;
 954         }
 955
 956         return !originalPortSecurityEnabled.equals(neutronPort.getPortSecurityEnabled());
 957     }
 958
 959     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 960         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 961         if (port1 == null) {
 962             return null;
 963         }
 964         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 965         if (port2 == null) {
 966             return list1;
 967         }
 968         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 969         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 970             NeutronSecurityGroup securityGroup1 = iterator.next();
 971             for (NeutronSecurityGroup securityGroup2 :list2) {
 972                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 973                     iterator.remove();
 974                 }
 975             }
 976         }
 977         return list1;
 978     }
 979
 980     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 981                                            String macAddress, String ipStr,
 982                                            Action actionForNode) {
 983         if (actionForNode == DELETE) {
 984             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 985                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 986         }
 987         if (actionForNode == ADD) {
 988             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 989                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 990         }
 991
 992         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 993                                                        macAddress, ipStr, actionForNode);
 994     }
 995
 996     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 997                                              String macAddress,
 998                                              String address,
 999                                              Action actionForNode) {
1000         Status status;
1001         try {
1002             InetAddress inetAddress = InetAddress.getByName(address);
1003             status = l3ForwardingProvider == null ?
1004                      new Status(StatusCode.SUCCESS) :
1005                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
1006                                                                       inetAddress, macAddress, actionForNode);
1007         } catch (UnknownHostException e) {
1008             status = new Status(StatusCode.BADREQUEST);
1009         }
1010
1011         if (status.isSuccess()) {
1012             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
1013                          l3ForwardingProvider == null ? "skipped" : "programmed",
1014                          macAddress, address, node.getNodeId().getValue(), actionForNode);
1015         } else {
1016             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
1017                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
1018         }
1019         return status;
1020     }
1021
1022     // --
1023
1024     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
1025                                                        Boolean isDelete) {
1026         Preconditions.checkNotNull(destNeutronRouterInterface);
1027
1028         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
1029         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
1030         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
1031         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
1032         final NeutronNetwork neutronNetwork = subnet != null ?
1033                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
1034         final String destinationSegmentationId = neutronNetwork != null ?
1035                                                  neutronNetwork.getProviderSegmentationID() : null;
1036         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
1037         final String cidr = subnet != null ? subnet.getCidr() : null;
1038         final int mask = getMaskLenFromCidr(cidr);
1039
1040         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
1041                      destNeutronRouterInterface, isDelete);
1042
1043         if (subnet != null && subnet.getIpVersion() == 6) {
1044             LOG.trace("programFlowsForNeutronRouterInterface doesn't support IPv6 router interface");
1045             return;
1046         }
1047         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
1048         // the local cache
1049         if (neutronNetwork != null) {
1050             if (macAddress == null || macAddress.isEmpty()) {
1051                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
1052             }
1053             if (ipList == null || ipList.isEmpty()) {
1054                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
1055             }
1056         }
1057
1058         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1059             cidr == null || cidr.isEmpty() ||
1060             macAddress == null || macAddress.isEmpty() ||
1061             ipList == null || ipList.isEmpty()) {
1062             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1063                          destinationSegmentationId, cidr, macAddress, ipList);
1064             // done: go no further w/out all the info needed...
1065             return;
1066         }
1067
1068         final Action actionForNode = isDelete ? DELETE : ADD;
1069
1070         // Keep cache for finding router's mac from network uuid -- add
1071         //
1072         if (! isDelete) {
1073             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1074             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1075             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1076         }
1077
1078         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1079         if (nodes.isEmpty()) {
1080             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1081         }
1082         for (Node node : nodes) {
1083             final Long dpid = getDpidForIntegrationBridge(node);
1084             if (dpid == null) {
1085                 continue;
1086             }
1087
1088             for (Neutron_IPs neutronIP : ipList) {
1089                 final String ipStr = neutronIP.getIpAddress();
1090                 if (ipStr.isEmpty()) {
1091                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1092                             node.getNodeId().getValue(), ipStr);
1093                     continue;
1094                 }
1095                 final IpAddress ipAddress = new IpAddress(ipStr.toCharArray());
1096                 if (ipAddress.getIpv4Address() == null) {
1097                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ipv6 {}",
1098                             node.getNodeId().getValue(), ipStr);
1099                     continue;
1100                 }
1101
1102                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1103                 //
1104                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1105                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1106                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1107                                                               neutronNetwork, destinationSegmentationId,
1108                                                               macAddress, ipStr, mask, actionForNode,
1109                                                               true /*isReflexsive*/);
1110                 }
1111
1112                 if (! isExternal) {
1113                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1114                             actionForNode);
1115                 }
1116                 // Enable ARP responder by default, because router interface needs to be responded always.
1117                 distributedArpService.programStaticRuleStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1118                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1119             }
1120
1121             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1122             // for the external neutron networks.
1123             //
1124             {
1125                 final Action actionForRewriteExclusion = isExternal ? DELETE : actionForNode;
1126                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1127             }
1128         }
1129
1130         if (isDelete) {
1131             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
1132             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
1133             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
1134         }
1135     }
1136
1137     private void programFlowForNetworkFromExternal(final Node node,
1138                                                    final Long dpid,
1139                                                    final String destinationSegmentationId,
1140                                                    final String dstMacAddress,
1141                                                    final String destIpStr,
1142                                                    final int destMask,
1143                                                    final Action actionForNode) {
1144         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1145                 dstMacAddress, destIpStr, destMask, actionForNode);
1146     }
1147
1148     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1149                                                            final Long dpid,
1150                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1151                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1152                                                            final NeutronNetwork dstNeutronNetwork,
1153                                                            final String destinationSegmentationId,
1154                                                            final String dstMacAddress,
1155                                                            final String destIpStr,
1156                                                            final int destMask,
1157                                                            final Action actionForNode,
1158                                                            Boolean isReflexsive) {
1159         Preconditions.checkNotNull(srcNeutronRouterInterface);
1160         Preconditions.checkNotNull(dstNeutronRouterInterface);
1161
1162         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1163         if (sourceSubnetId == null) {
1164             LOG.error("Could not get provider Subnet ID from router interface {}",
1165                          srcNeutronRouterInterface.getID());
1166             return;
1167         }
1168
1169         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1170         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1171         if (sourceNetworkId == null) {
1172             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1173             return;
1174         }
1175
1176         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1177         if (sourceNetwork == null) {
1178             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1179             return;
1180         }
1181
1182         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1183             // Isolate subnets from different tenants within the same router
1184             return;
1185         }
1186         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1187         if (sourceSegmentationId == null) {
1188             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1189             return;
1190         }
1191         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1192             // Skip 'self'
1193             return;
1194         }
1195
1196         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1197                                      dstMacAddress, destIpStr, destMask, actionForNode);
1198
1199         // Flip roles src->dst; dst->src
1200         if (isReflexsive) {
1201             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1202             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1203             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1204             final String cidr2 = sourceSubnet.getCidr();
1205             final int mask2 = getMaskLenFromCidr(cidr2);
1206
1207             if (cidr2 == null || cidr2.isEmpty() ||
1208                 macAddress2 == null || macAddress2.isEmpty() ||
1209                 ipList2 == null || ipList2.isEmpty()) {
1210                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1211                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1212                 // done: go no further w/out all the info needed...
1213                 return;
1214             }
1215
1216             for (Neutron_IPs neutronIP2 : ipList2) {
1217                 final String ipStr2 = neutronIP2.getIpAddress();
1218                 if (ipStr2.isEmpty()) {
1219                     continue;
1220                 }
1221                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1222                                                           srcNeutronRouterInterface,
1223                                                           sourceNetwork, sourceSegmentationId,
1224                                                           macAddress2, ipStr2, mask2, actionForNode,
1225                                                           false /*isReflexsive*/);
1226             }
1227         }
1228     }
1229
1230     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1231                                               String destinationSegmentationId,
1232                                               String macAddress, String ipStr, int mask,
1233                                               Action actionForNode) {
1234         if (actionForNode == DELETE) {
1235             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1236                          " action {}",
1237                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1238                          macAddress, ipStr, mask, actionForNode);
1239         }
1240         if (actionForNode == ADD) {
1241             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1242                          " action {}",
1243                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1244                          macAddress, ipStr, mask, actionForNode);
1245         }
1246
1247         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1248                                                           macAddress, ipStr, mask, actionForNode);
1249     }
1250
1251     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1252                                                 String destinationSegmentationId,
1253                                                 String macAddress,
1254                                                 String address, int mask,
1255                                                 Action actionForNode) {
1256         Status status;
1257         try {
1258             InetAddress inetAddress = InetAddress.getByName(address);
1259             status = routingProvider == null ?
1260                      new Status(StatusCode.SUCCESS) :
1261                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1262                                                             macAddress, inetAddress, mask, actionForNode);
1263         } catch (UnknownHostException e) {
1264             status = new Status(StatusCode.BADREQUEST);
1265         }
1266
1267         if (status.isSuccess()) {
1268             LOG.debug("programRouterInterfaceStage2 {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1269                          routingProvider == null ? "skipped" : "programmed",
1270                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1271                          actionForNode);
1272         } else {
1273             LOG.error("programRouterInterfaceStage2 failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1274                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1275                          actionForNode, status);
1276         }
1277         return status;
1278     }
1279
1280     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1281                                            String macAddress, String ipStr,
1282                                            Action action) {
1283         if (action == DELETE ) {
1284             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1285                     dpid, segOrOfPort, macAddress, ipStr, action);
1286         }
1287         if (action == ADD) {
1288             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1289                     dpid, segOrOfPort, macAddress, ipStr, action);
1290         }
1291
1292         Status status = new Status(StatusCode.UNSUPPORTED);
1293         if (icmpEchoProvider != null){
1294             try {
1295                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1296                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1297                                                 macAddress, inetAddress, action);
1298             } catch (UnknownHostException e) {
1299                 status = new Status(StatusCode.BADREQUEST);
1300             }
1301         }
1302
1303         if (status.isSuccess()) {
1304             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1305                     icmpEchoProvider == null ? "skipped" : "programmed",
1306                     macAddress, ipStr, dpid, segOrOfPort, action);
1307         } else {
1308             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1309                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1310         }
1311
1312         return status.isSuccess();
1313     }
1314
1315     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1316                                                   String matchAddress, String rewriteAddress,
1317                                                   Action action) {
1318         if (action == DELETE ) {
1319             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1320                     " action {}",
1321                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1322         }
1323         if (action == ADD ) {
1324             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1325                     " action {}",
1326                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1327         }
1328
1329         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1330                 rewriteAddress, action);
1331         return status.isSuccess();
1332     }
1333
1334     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1335                                                  String matchAddress, String rewriteAddress,
1336                                                  Action action) {
1337         Status status;
1338         try {
1339             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1340             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1341             status = inboundNatProvider == null ?
1342                     new Status(StatusCode.SUCCESS) :
1343                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1344                             inetMatchAddress, inetRewriteAddress,
1345                             action);
1346         } catch (UnknownHostException e) {
1347             status = new Status(StatusCode.BADREQUEST);
1348         }
1349
1350         if (status.isSuccess()) {
1351             final boolean isSkipped = inboundNatProvider == null;
1352             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1353                     isSkipped ? "skipped" : "programmed",
1354                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1355         } else {
1356             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1357                          " status:{}",
1358                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1359                     status);
1360         }
1361         return status;
1362     }
1363
1364     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1365                                                  Action actionForRewriteExclusion) {
1366         if (actionForRewriteExclusion == DELETE ) {
1367             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1368                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1369         }
1370         if (actionForRewriteExclusion == ADD) {
1371             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1372                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1373         }
1374
1375         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1376     }
1377
1378     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1379                                                    Action actionForNode) {
1380         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1381                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1382
1383         if (status.isSuccess()) {
1384             final boolean isSkipped = outboundNatProvider == null;
1385             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1386                          isSkipped ? "skipped" : "programmed",
1387                          cidr, node.getNodeId().getValue(), actionForNode);
1388         } else {
1389             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1390                          cidr, node.getNodeId().getValue(), actionForNode, status);
1391         }
1392         return status;
1393     }
1394
1395     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1396
1397         if (action == DELETE) {
1398             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1399                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1400         }
1401         if (action == ADD) {
1402             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1403                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1404         }
1405
1406         this.programOutboundIpRewriteStage2(fid, action);
1407     }
1408
1409     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1410         Status status;
1411         try {
1412             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1413             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1414             status = outboundNatProvider == null ?
1415                     new Status(StatusCode.SUCCESS) :
1416                     outboundNatProvider.programIpRewriteRule(
1417                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1418                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1419         } catch (UnknownHostException e) {
1420             status = new Status(StatusCode.BADREQUEST);
1421         }
1422
1423         if (status.isSuccess()) {
1424             final boolean isSkipped = outboundNatProvider == null;
1425             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1426                             " action {}",
1427                          isSkipped ? "skipped" : "programmed",
1428                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1429         } else {
1430             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1431                          " action {} status:{}",
1432                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1433         }
1434         return status;
1435     }
1436
1437     private int getMaskLenFromCidr(String cidr) {
1438         if (cidr == null) {
1439             return 0;
1440         }
1441         String[] splits = cidr.split("/");
1442         if (splits.length != 2) {
1443             return 0;
1444         }
1445
1446         int result;
1447         try {
1448             result = Integer.parseInt(splits[1].trim());
1449         } catch (NumberFormatException nfe) {
1450             result = 0;
1451         }
1452         return result;
1453     }
1454
1455     private Long getDpidForIntegrationBridge(Node node) {
1456         // Check if node is integration bridge; and only then return its dpid
1457         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1458             return southbound.getDataPathId(node);
1459         }
1460         return null;
1461     }
1462
1463     private Long getDpidForExternalBridge(Node node) {
1464         // Check if node is external bridge; and only then return its dpid
1465         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1466             return southbound.getDataPathId(node);
1467         }
1468         return null;
1469     }
1470
1471     private Node getExternalBridgeNode(){
1472         //Pickup the first node that has external bridge (br-ex).
1473         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1474         //the external network is reachable from every br-ex
1475         // TODO: Consider other deployment scenario, and thing of better solution.
1476         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1477         for(Node node : allBridges){
1478             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1479                 return node;
1480             }
1481         }
1482         return null;
1483     }
1484
1485     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1486         if (gatewayPort.getFixedIPs() == null) {
1487             return null;
1488         }
1489         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1490             String subnetUUID = neutronIPs.getSubnetUUID();
1491             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1492             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1493                 return extSubnet;
1494             }
1495             if (extSubnet == null) {
1496                 // TODO: when subnet is created, try again.
1497                 LOG.debug("subnet {} in not found", subnetUUID);
1498              }
1499         }
1500         return null;
1501     }
1502
1503     private void cleanupFloatingIPRules(final NeutronPort neutronPort) {
1504         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1505         if (neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
1506             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1507                 // Neutron floating Ip's port uuid cannot be null (Bug#5894)
1508                 if (neutronFloatingIP.getPortUUID() != null &&
1509                         (neutronFloatingIP.getPortUUID().equals(neutronPort.getPortUUID()))) {
1510                     handleNeutronFloatingIPEvent(neutronFloatingIP, DELETE);
1511                 }
1512             }
1513         }
1514     }
1515
1516     private void updateFloatingIPRules(final NeutronPort neutronPort) {
1517         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
1518         if (neutronFloatingIps != null) {
1519             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
1520                 if (neutronFloatingIP.getPortUUID() != null
1521                         && neutronFloatingIP.getPortUUID().equals(
1522                                 neutronPort.getPortUUID())) {
1523                     handleNeutronFloatingIPEvent(neutronFloatingIP, UPDATE);
1524                 }
1525             }
1526         }
1527     }
1528
1529
1530     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1531
1532         Preconditions.checkNotNull(gatewayPort);
1533         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1534
1535         if(externalNetwork != null){
1536             if(externalNetwork.isRouterExternal()){
1537                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1538
1539                 // TODO: address IPv6 case.
1540                 if (externalSubnet != null &&
1541                         externalSubnet.getIpVersion() == 4 &&
1542                         gatewayPort.getFixedIPs() != null) {
1543                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1544                     Neutron_IPs neutronIP = null;
1545                     for (Neutron_IPs nIP : gatewayPort.getFixedIPs()) {
1546                         InetAddress ipAddress;
1547                         try {
1548                             ipAddress = InetAddress.getByName(nIP.getIpAddress());
1549                         } catch (UnknownHostException e) {
1550                             LOG.warn("unknown host exception {}", e);
1551                             continue;
1552                         }
1553                         if (ipAddress instanceof Inet4Address) {
1554                             neutronIP = nIP;
1555                             break;
1556                         }
1557                     }
1558                     if (neutronIP == null) {
1559                         // TODO IPv6 neighbor discovery
1560                         LOG.debug("Ignoring gateway ports with IPv6 only fixed ip {}",
1561                                   gatewayPort.getFixedIPs());
1562                     } else {
1563                         gatewayMacResolver.resolveMacAddress(
1564                             this, /* gatewayMacResolverListener */
1565                             null, /* externalNetworkBridgeDpid */
1566                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1567                             new Ipv4Address(externalSubnet.getGatewayIP()),
1568                             new Ipv4Address(neutronIP.getIpAddress()),
1569                             new MacAddress(gatewayPort.getMacAddress()),
1570                             true /* periodicRefresh */);
1571                     }
1572                 } else {
1573                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1574                 }
1575             }
1576         }else{
1577             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1578         }
1579     }
1580
1581
1582     private void storePortInCleanupCache(NeutronPort port) {
1583         this.portCleanupCache.put(port.getPortUUID(),port);
1584     }
1585
1586
1587     private void updatePortInCleanupCache(NeutronPort updatedPort,NeutronPort originalPort) {
1588         removePortFromCleanupCache(originalPort);
1589         storePortInCleanupCache(updatedPort);
1590     }
1591
1592     public void removePortFromCleanupCache(NeutronPort port) {
1593         if(port != null) {
1594             this.portCleanupCache.remove(port.getPortUUID());
1595         }
1596     }
1597
1598     public Map<String, NeutronPort> getPortCleanupCache() {
1599         return this.portCleanupCache;
1600     }
1601
1602     public NeutronPort getPortFromCleanupCache(String portid) {
1603         NeutronPort ret = this.portCleanupCache.get(portid);
1604         if (ret != null) {
1605             LOG.info("getPortFromCleanupCache: Matching NeutronPort found {}", portid);
1606         }
1607         return ret;
1608     }
1609
1610     private void storeNetworkInCleanupCache(NeutronNetwork network) {
1611         if (network != null) {
1612             this.networkCleanupCache.put(network.getNetworkUUID(), network);
1613         }
1614     }
1615
1616
1617     private void updateNetworkInCleanupCache(NeutronNetwork network) {
1618         this.networkCleanupCache.put(network.getNetworkUUID(), network);
1619     }
1620
1621     public void removeNetworkFromCleanupCache(String networkid) {
1622         NeutronNetwork network = networkCleanupCache.get(networkid);
1623         if (network != null) {
1624             for (NeutronPort port : this.portCleanupCache.values()) {
1625                 if (port.getNetworkUUID().equals(networkid)) {
1626                     LOG.info("This network is used by another port", network);
1627                     return;
1628                 }
1629             }
1630             this.networkCleanupCache.remove(network.getNetworkUUID());
1631         }
1632     }
1633
1634     public NeutronPort getPortPreferablyFromCleanupCache(String portUuid) {
1635         NeutronPort port = getPortFromCleanupCache(portUuid);
1636         if (port == null) {
1637             port = neutronPortCache.getPort(portUuid);
1638             if (port == null) {
1639                 LOG.warn("In getPortPreferablyFromCleanupCache no neutron port found:  portUuid: {}", portUuid);
1640             }
1641         }
1642         return port;
1643     }
1644
1645     public Map<String, NeutronNetwork> getNetworkCleanupCache() {
1646         return this.networkCleanupCache;
1647     }
1648
1649     public NeutronNetwork getNetworkFromCleanupCache(String networkid) {
1650         NeutronNetwork ret = networkCleanupCache.get(networkid);
1651         if (ret != null) {
1652             LOG.info("getNetworkFromCleanupCache: Matching NeutronNetwork found {}", networkid);
1653         }
1654         return ret;
1655     }
1656     /**
1657      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1658      *
1659      * @param ofPort the OF port number
1660      * @return the string with encoded OF port (example format "OFPort|999")
1661      */
1662     public static String encodeExcplicitOFPort(Long ofPort) {
1663         return "OFPort|" + ofPort.toString();
1664     }
1665     private void initNetworkCleanUpCache() {
1666         if (this.neutronNetworkCache != null) {
1667             for (NeutronNetwork neutronNetwork : neutronNetworkCache.getAllNetworks()) {
1668                 networkCleanupCache.put(neutronNetwork.getNetworkUUID(), neutronNetwork);
1669             }
1670         }
1671     }
1672     private void initPortCleanUpCache() {
1673         if (this.neutronPortCache != null) {
1674             for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
1675                 portCleanupCache.put(neutronPort.getPortUUID(), neutronPort);
1676             }
1677         }
1678     }
1679     @Override
1680     public void setDependencies(ServiceReference serviceReference) {
1681         eventDispatcher =
1682                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1683         eventDispatcher.eventHandlerAdded(serviceReference, this);
1684         tenantNetworkManager =
1685                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1686         configurationService =
1687                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1688         arpProvider =
1689                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1690         inboundNatProvider =
1691                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1692         outboundNatProvider =
1693                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1694         routingProvider =
1695                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1696         l3ForwardingProvider =
1697                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1698         distributedArpService =
1699                  (DistributedArpService) ServiceHelper.getGlobalInstance(DistributedArpService.class, this);
1700         nodeCacheManager =
1701                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1702         southbound =
1703                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1704         gatewayMacResolver =
1705                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1706         securityServicesManager =
1707                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1708
1709         initL3AdapterMembers();
1710     }
1711
1712     @Override
1713     public void setDependencies(Object impl) {
1714         if (impl instanceof INeutronNetworkCRUD) {
1715             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1716             initNetworkCleanUpCache();
1717         } else if (impl instanceof INeutronPortCRUD) {
1718             neutronPortCache = (INeutronPortCRUD)impl;
1719             initPortCleanUpCache();
1720         } else if (impl instanceof INeutronSubnetCRUD) {
1721             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1722         } else if (impl instanceof INeutronFloatingIPCRUD) {
1723             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1724         } else if (impl instanceof ArpProvider) {
1725             arpProvider = (ArpProvider)impl;
1726         } else if (impl instanceof InboundNatProvider) {
1727             inboundNatProvider = (InboundNatProvider)impl;
1728         } else if (impl instanceof OutboundNatProvider) {
1729             outboundNatProvider = (OutboundNatProvider)impl;
1730         } else if (impl instanceof RoutingProvider) {
1731             routingProvider = (RoutingProvider)impl;
1732         } else if (impl instanceof L3ForwardingProvider) {
1733             l3ForwardingProvider = (L3ForwardingProvider)impl;
1734         }else if (impl instanceof GatewayMacResolver) {
1735             gatewayMacResolver = (GatewayMacResolver)impl;
1736         }else if (impl instanceof IcmpEchoProvider) {
1737             icmpEchoProvider = (IcmpEchoProvider)impl;
1738         }
1739
1740         populateL3ForwardingCaches();
1741     }
1742 }