openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (c) 2014 - 2016 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  10
  11 import com.google.common.base.Preconditions;
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.ovsdb.openstack.netvirt.AbstractEvent;
  15 import org.opendaylight.ovsdb.openstack.netvirt.AbstractHandler;
  16 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  17 import org.opendaylight.ovsdb.openstack.netvirt.NeutronL3AdapterEvent;
  18 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  19 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  20 import org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService;
  21 import org.opendaylight.ovsdb.openstack.netvirt.api.Constants;
  22 import org.opendaylight.ovsdb.openstack.netvirt.api.EventDispatcher;
  23 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolver;
  24 import org.opendaylight.ovsdb.openstack.netvirt.api.GatewayMacResolverListener;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.IcmpEchoProvider;
  26 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  27 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.NodeCacheManager;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  31 import org.opendaylight.ovsdb.openstack.netvirt.api.SecurityServicesManager;
  32 import org.opendaylight.ovsdb.openstack.netvirt.api.Southbound;
  33 import org.opendaylight.ovsdb.openstack.netvirt.api.Status;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.StatusCode;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  36 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP;
  37 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork;
  38 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronPort;
  39 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter;
  40 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronRouter_Interface;
  41 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSecurityGroup;
  42 import org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronSubnet;
  43 import org.opendaylight.ovsdb.openstack.netvirt.translator.Neutron_IPs;
  44 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronFloatingIPCRUD;
  45 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronNetworkCRUD;
  46 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronPortCRUD;
  47 import org.opendaylight.ovsdb.openstack.netvirt.translator.crud.INeutronSubnetCRUD;
  48 import org.opendaylight.ovsdb.openstack.netvirt.translator.iaware.impl.NeutronIAwareUtil;
  49 import org.opendaylight.ovsdb.utils.neutron.utils.NeutronModelsDataStoreHelper;
  50 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  51 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.IpAddress;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev100924.Ipv4Address;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev100924.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.Routers;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l3.rev150712.routers.attributes.routers.Router;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  60 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  61 import org.osgi.framework.ServiceReference;
  62 import org.slf4j.Logger;
  63 import org.slf4j.LoggerFactory;
  64
  65 import java.net.InetAddress;
  66 import java.net.UnknownHostException;
  67 import java.util.ArrayList;
  68 import java.util.Collection;
  69 import java.util.HashMap;
  70 import java.util.Iterator;
  71 import java.util.List;
  72 import java.util.Map;
  73
  74 /**
  75  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  76  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  77  * as well as the multi-tenant router forwarding provider.
  78  */
  79 public class NeutronL3Adapter extends AbstractHandler implements GatewayMacResolverListener, ConfigInterface {
  80     private static final Logger LOG = LoggerFactory.getLogger(NeutronL3Adapter.class);
  81
  82     // The implementation for each of these services is resolved by the OSGi Service Manager
  83     private volatile ConfigurationService configurationService;
  84     private volatile TenantNetworkManager tenantNetworkManager;
  85     private volatile NodeCacheManager nodeCacheManager;
  86     private volatile INeutronNetworkCRUD neutronNetworkCache;
  87     private volatile INeutronSubnetCRUD neutronSubnetCache;
  88     private volatile INeutronPortCRUD neutronPortCache;
  89     private volatile INeutronFloatingIPCRUD neutronFloatingIpCache;
  90     private volatile L3ForwardingProvider l3ForwardingProvider;
  91     private volatile InboundNatProvider inboundNatProvider;
  92     private volatile OutboundNatProvider outboundNatProvider;
  93     private volatile ArpProvider arpProvider;
  94     private volatile RoutingProvider routingProvider;
  95     private volatile GatewayMacResolver gatewayMacResolver;
  96     private volatile SecurityServicesManager securityServicesManager;
  97     private volatile IcmpEchoProvider icmpEchoProvider;
  98
  99     private class FloatIpData {
 100         // br-int of node where floating ip is associated with tenant port
 101         private final Long dpid;
 102         // patch port in br-int used to reach br-ex
 103         private final Long ofPort;
 104         // segmentation id of the net where fixed ip is instantiated
 105         private final String segId;
 106         // mac address assigned to neutron port of floating ip
 107         private final String macAddress;
 108         private final String floatingIpAddress;
 109         // ip address given to tenant vm
 110         private final String fixedIpAddress;
 111         private final String neutronRouterMac;
 112
 113         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
 114                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
 115             this.dpid = dpid;
 116             this.ofPort = ofPort;
 117             this.segId = segId;
 118             this.macAddress = macAddress;
 119             this.floatingIpAddress = floatingIpAddress;
 120             this.fixedIpAddress = fixedIpAddress;
 121             this.neutronRouterMac = neutronRouterMac;
 122         }
 123     }
 124
 125     private Map<String, String> networkIdToRouterMacCache;
 126     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
 127     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
 128
 129     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
 130     private Map<String, FloatIpData> floatIpDataMapCache;
 131
 132     private String externalRouterMac;
 133     private Boolean enabled = false;
 134     private Boolean flgDistributedARPEnabled = true;
 135     private Boolean isCachePopulationDone = false;
 136
 137     private Southbound southbound;
 138     private NeutronModelsDataStoreHelper neutronModelsDataStoreHelper;
 139
 140     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 141     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 142     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 143     private static final String OWNER_FLOATING_IP = "network:floatingip";
 144     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 145
 146     public NeutronL3Adapter(NeutronModelsDataStoreHelper neutronHelper) {
 147         LOG.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 148         this.neutronModelsDataStoreHelper = neutronHelper;
 149     }
 150
 151     private void initL3AdapterMembers() {
 152         Preconditions.checkNotNull(configurationService);
 153
 154         if (configurationService.isL3ForwardingEnabled()) {
 155             this.networkIdToRouterMacCache = new HashMap<>();
 156             this.networkIdToRouterIpListCache = new HashMap<>();
 157             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 158             this.neutronPortToDpIdCache = new HashMap<>();
 159             this.floatIpDataMapCache = new HashMap<>();
 160
 161             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 162             if (this.externalRouterMac == null) {
 163                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 164             }
 165
 166             this.enabled = true;
 167             LOG.info("OVSDB L3 forwarding is enabled");
 168             if (configurationService.isDistributedArpDisabled()) {
 169                 this.flgDistributedARPEnabled = false;
 170                 LOG.info("Distributed ARP responder is disabled");
 171             } else {
 172                 LOG.debug("Distributed ARP responder is enabled");
 173             }
 174         } else {
 175             LOG.debug("OVSDB L3 forwarding is disabled");
 176         }
 177     }
 178
 179     //
 180     // Callbacks from AbstractHandler
 181     //
 182     @Override
 183     public void processEvent(AbstractEvent abstractEvent) {
 184         if (!(abstractEvent instanceof NeutronL3AdapterEvent)) {
 185             LOG.error("Unable to process abstract event " + abstractEvent);
 186             return;
 187         }
 188         if (!this.enabled) {
 189             return;
 190         }
 191
 192         NeutronL3AdapterEvent ev = (NeutronL3AdapterEvent) abstractEvent;
 193         switch (ev.getAction()) {
 194             case UPDATE:
 195                 if (ev.getSubType() == NeutronL3AdapterEvent.SubType.SUBTYPE_EXTERNAL_MAC_UPDATE) {
 196                     updateExternalRouterMac( ev.getMacAddress().getValue() );
 197                 } else {
 198                     LOG.warn("Received update for an unexpected event " + ev);
 199                 }
 200                 break;
 201             case ADD:
 202                 // fall through...
 203                 // break;
 204             case DELETE:
 205                 // fall through...
 206                 // break;
 207             default:
 208                 LOG.warn("Unable to process event " + ev);
 209                 break;
 210         }
 211     }
 212
 213     //
 214     // Callbacks from GatewayMacResolverListener
 215     //
 216
 217     @Override
 218     public void gatewayMacResolved(Long externalNetworkBridgeDpid, IpAddress gatewayIpAddress, MacAddress macAddress) {
 219         LOG.info("got gatewayMacResolved callback for ip {} on dpid {} to mac {}",
 220                 gatewayIpAddress, externalNetworkBridgeDpid, macAddress);
 221         if (!this.enabled) {
 222             return;
 223         }
 224
 225         if (macAddress == null || macAddress.getValue() == null) {
 226             // TODO: handle cases when mac is null
 227             return;
 228         }
 229
 230         //
 231         // Enqueue event so update is handled by adapter's thread
 232         //
 233         enqueueEvent( new NeutronL3AdapterEvent(externalNetworkBridgeDpid, gatewayIpAddress, macAddress) );
 234     }
 235
 236     private void populateL3ForwardingCaches() {
 237         if (!this.enabled) {
 238             return;
 239         }
 240         if(this.isCachePopulationDone || this.neutronFloatingIpCache == null
 241                 || this.neutronPortCache == null ||this.neutronNetworkCache == null) {
 242             return;
 243         }
 244         this.isCachePopulationDone = true;
 245         LOG.debug("Populating NetVirt L3 caches from data store configuration");
 246         Routers routers = this.neutronModelsDataStoreHelper.readAllNeutronRouters();
 247         Ports ports = this.neutronModelsDataStoreHelper.readAllNeutronPorts();
 248         if(routers != null && routers.getRouter() != null && ports != null) {
 249             LOG.debug("L3 Cache Population : {} Neutron router present in data store",routers.getRouter().size());
 250             for( Router router : routers.getRouter()) {
 251                 LOG.debug("L3 Cache Population : Populate caches for router {}",router);
 252                 if(!ports.getPort().isEmpty()) {
 253                     for( Port port : ports.getPort()) {
 254                         if (port.getDeviceId().equals(router.getUuid().getValue()) &&
 255                                 port.getDeviceOwner().equals(OWNER_ROUTER_INTERFACE)) {
 256                             LOG.debug("L3 Cache Population : Router interface {} found.",port);
 257                             networkIdToRouterMacCache.put(port.getNetworkId().getValue()
 258                                     , port.getMacAddress());
 259
 260                             networkIdToRouterIpListCache.put(port.getNetworkId().getValue(),
 261                                     NeutronIAwareUtil.convertMDSalIpToNeutronIp(port.getFixedIps()));
 262                             subnetIdToRouterInterfaceCache.put(port.getFixedIps().get(0).getSubnetId().getValue(),
 263                                     NeutronIAwareUtil.convertMDSalInterfaceToNeutronRouterInterface(port));
 264                         }
 265                     }
 266                 }else {
 267                     LOG.warn("L3 Cache Population :Did not find any port information " +
 268                             "in config Data Store for router {}",router);
 269                 }
 270             }
 271         }
 272         LOG.debug("NetVirt L3 caches population is done");
 273     }
 274
 275     private Pair<Long, Uuid> getDpIdOfNeutronPort(String neutronTenantPortUuid) {
 276         if(neutronPortToDpIdCache.get(neutronTenantPortUuid) == null) {
 277             List<Node> bridges = this.southbound.readOvsdbTopologyBridgeNodes();
 278             LOG.debug("getDpIdOfNeutronPort : {} bridges present in ovsdb topology",bridges.size());
 279             for(Node bridge : bridges) {
 280                 List<OvsdbTerminationPointAugmentation> interfaces
 281                         = southbound.extractTerminationPointAugmentations(bridge);
 282                 if(interfaces != null && !interfaces.isEmpty()) {
 283                     LOG.debug("getDpIdOfNeutronPort : {} termination point present on bridge {}",
 284                             interfaces.size(), bridge.getNodeId());
 285                     for (OvsdbTerminationPointAugmentation intf : interfaces) {
 286                         NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 287                         if(neutronPort != null && neutronPort.getID().equals(neutronTenantPortUuid)) {
 288                             Long dpId = getDpidForIntegrationBridge(bridge);
 289                             Uuid interfaceUuid = intf.getInterfaceUuid();
 290                             LOG.debug("getDpIdOfNeutronPort : Found bridge {} and interface {} for the tenant neutron" +
 291                                     " port {}",dpId,interfaceUuid,neutronTenantPortUuid);
 292                             handleInterfaceEventAdd(neutronPort.getPortUUID(), dpId, interfaceUuid);
 293                             break;
 294                         }
 295                     }
 296                 }
 297             }
 298         }
 299         return neutronPortToDpIdCache.get(neutronTenantPortUuid);
 300     }
 301
 302     private Collection<FloatIpData> getAllFloatingIPsWithMetadata() {
 303         LOG.debug("getAllFloatingIPsWithMetadata : Fechting all floating Ips and it's metadata");
 304         List<NeutronFloatingIP> neutronFloatingIps = neutronFloatingIpCache.getAllFloatingIPs();
 305         if(neutronFloatingIps != null && !neutronFloatingIps.isEmpty()) {
 306             for (NeutronFloatingIP neutronFloatingIP : neutronFloatingIps) {
 307                 if(!floatIpDataMapCache.containsKey(neutronFloatingIP.getID())){
 308                     LOG.debug("Metadata for floating ip {} is not present in the cache. " +
 309                             "Fetching from data store.",neutronFloatingIP.getID());
 310                     this.getFloatingIPWithMetadata(neutronFloatingIP.getID());
 311                 }
 312             }
 313         }
 314         LOG.debug("getAllFloatingIPsWithMetadata : {} floating points found in data store",floatIpDataMapCache.size());
 315         return floatIpDataMapCache.values();
 316     }
 317     private FloatIpData getFloatingIPWithMetadata(String neutronFloatingId) {
 318         LOG.debug("getFloatingIPWithMetadata : Get Floating ip and it's meta data for neutron " +
 319                 "floating id {} ",neutronFloatingId);
 320         if(floatIpDataMapCache.get(neutronFloatingId) == null) {
 321             NeutronFloatingIP neutronFloatingIP = neutronFloatingIpCache.getFloatingIP(neutronFloatingId);
 322             if (neutronFloatingIP == null) {
 323                 LOG.error("getFloatingIPWithMetadata : Floating ip {} is missing from data store, that should not happen",neutronFloatingId);
 324                 return null;
 325             }
 326             List<NeutronPort> neutronPorts = neutronPortCache.getAllPorts();
 327             NeutronPort neutronPortForFloatIp = null;
 328             for (NeutronPort neutronPort : neutronPorts) {
 329                 if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 330                         neutronPort.getDeviceID().equals(neutronFloatingIP.getID())) {
 331                     neutronPortForFloatIp = neutronPort;
 332                     break;
 333                 }
 334             }
 335
 336             String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 337             if(neutronTenantPortUuid == null) {
 338                 return null;
 339             }
 340             Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 341             String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 342             String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 343             String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 344
 345             NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 346             NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 347                     neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 348             String providerSegmentationId = tenantNeutronNetwork != null ?
 349                     tenantNeutronNetwork.getProviderSegmentationID() : null;
 350             String neutronRouterMac = tenantNeutronNetwork != null ?
 351                     networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 352
 353             if (nodeIfPair == null || neutronTenantPortUuid == null ||
 354                     providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 355                     floatingIpMac == null || floatingIpMac.isEmpty() ||
 356                     neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 357                 LOG.debug("getFloatingIPWithMetadata :Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} " +
 358                                 "seg {} mac {} rtrMac {}",
 359                         fixedIpAddress,
 360                         floatingIpAddress,
 361                         neutronPortForFloatIp,
 362                         neutronTenantPortUuid,
 363                         providerSegmentationId,
 364                         floatingIpMac,
 365                         neutronRouterMac);
 366
 367                 return null;
 368             }
 369
 370             // get ofport for patch port in br-int
 371             final Long dpId = nodeIfPair.getLeft();
 372             final Long ofPort = findOFPortForExtPatch(dpId);
 373             if (ofPort == null) {
 374                 LOG.warn("getFloatingIPWithMetadata : Unable to locate OF port of patch port " +
 375                                 "to connect floating ip to external bridge. dpid {}",
 376                         dpId);
 377                 return null;
 378             }
 379
 380             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 381                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 382             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 383
 384         }
 385         return floatIpDataMapCache.get(neutronFloatingId);
 386     }
 387     /**
 388      * Invoked to configure the mac address for the external gateway in br-ex. ovsdb netvirt needs help in getting
 389      * mac for given ip in br-ex (bug 3378). For example, since ovsdb has no real arp, it needs a service in can
 390      * subscribe so that the mac address associated to the gateway ip address is available.
 391      *
 392      * @param externalRouterMacUpdate  The mac address to be associated to the gateway.
 393      */
 394     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 395         Preconditions.checkNotNull(externalRouterMacUpdate);
 396
 397         flushExistingIpRewrite();
 398         this.externalRouterMac = externalRouterMacUpdate;
 399         rebuildExistingIpRewrite();
 400     }
 401
 402     /**
 403      * Process the event.
 404      *
 405      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 406      * @param subnet An instance of NeutronSubnet object.
 407      */
 408     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 409         LOG.debug("Neutron subnet {} event : {}", action, subnet.toString());
 410     }
 411
 412     /**
 413      * Process the port event as a router interface event.
 414      * For a not delete action, since a port is only create when the tennat uses the subnet, it is required to
 415      * verify if all routers across all nodes have the interface for the port's subnet(s) configured.
 416      *
 417      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 418      * @param neutronPort An instance of NeutronPort object.
 419      */
 420     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 421         LOG.debug("Neutron port {} event : {}", action, neutronPort.toString());
 422
 423         if (!this.enabled) {
 424             return;
 425         }
 426         this.processSecurityGroupUpdate(neutronPort);
 427
 428         final boolean isDelete = action == Action.DELETE;
 429
 430         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_GATEWAY)){
 431             if (!isDelete) {
 432                 LOG.info("Port {} is network router gateway interface, "
 433                         + "triggering gateway resolution for the attached external network", neutronPort);
 434                 this.triggerGatewayMacResolver(neutronPort);
 435             }else{
 436                 NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(neutronPort.getNetworkUUID());
 437
 438                 if (externalNetwork != null && externalNetwork.isRouterExternal()) {
 439                     final NeutronSubnet externalSubnet = getExternalNetworkSubnet(neutronPort);
 440                     // TODO support IPv6
 441                     if (externalSubnet != null &&
 442                             externalSubnet.getIpVersion() == 4) {
 443                         gatewayMacResolver.stopPeriodicRefresh(new Ipv4Address(externalSubnet.getGatewayIP()));
 444                     }
 445                 }
 446             }
 447         }
 448
 449         // Treat the port event as a router interface event if the port belongs to router. This is a
 450         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 451         //
 452         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 453             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 454
 455             if (neutronPort.getFixedIPs() != null) {
 456                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 457                     NeutronRouter_Interface neutronRouterInterface =
 458                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 459                     // id of router interface to be same as subnet
 460                     neutronRouterInterface.setID(neutronIP.getSubnetUUID());
 461                     neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 462
 463                     this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 464                 }
 465             }
 466         } else {
 467             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 468             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 469             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 470             // there.
 471             //
 472             if (!isDelete && neutronPort.getFixedIPs() != null) {
 473                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 474                     NeutronRouter_Interface neutronRouterInterface =
 475                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 476                     if (neutronRouterInterface != null) {
 477                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 478                     }
 479                 }
 480             }
 481             this.updateL3ForNeutronPort(neutronPort, isDelete);
 482         }
 483     }
 484
 485     /**
 486      * Process the event.
 487      *
 488      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 489      * @param neutronRouter An instance of NeutronRouter object.
 490      */
 491     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 492         LOG.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 493     }
 494
 495     /**
 496      * Process the event enforcing actions and verifying dependencies between all router's interface. For example,
 497      * delete the ports on the same subnet.
 498      *
 499      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 500      * @param neutronRouter An instance of NeutronRouter object.
 501      * @param neutronRouterInterface An instance of NeutronRouter_Interface object.
 502      */
 503     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 504                                                   final NeutronRouter_Interface neutronRouterInterface,
 505                                                   Action action) {
 506         LOG.debug("Router interface {} got event {}. Subnet {}",
 507                      neutronRouterInterface.getPortUUID(),
 508                      action,
 509                      neutronRouterInterface.getSubnetUUID());
 510         if (!this.enabled) {
 511             return;
 512         }
 513
 514         final boolean isDelete = action == Action.DELETE;
 515
 516         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 517
 518         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 519         // see if they are affected by l3
 520         //
 521         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 522             boolean currPortShouldBeDeleted = false;
 523             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 524             if (isDelete) {
 525                 if (neutronPort.getFixedIPs() != null) {
 526                     for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 527                         if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 528                             currPortShouldBeDeleted = true;
 529                             break;
 530                         }
 531                     }
 532                 }
 533             }
 534             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 535         }
 536
 537         if (isDelete) {
 538             /*
 539              *  Bug 4277: Remove the router interface cache only after deleting the neutron port l3 flows.
 540              */
 541             this.cleanupRouterCache(neutronRouterInterface);
 542         }
 543     }
 544
 545     /**
 546      * Invoked when a neutron message regarding the floating ip association is sent to odl via ml2. If the action is
 547      * a creation, it will first add ARP rules for the given floating ip and then configure the DNAT (rewrite the
 548      * packets from the floating IP address to the internal fixed ip) rules on OpenFlow Table 30 and SNAT rules (other
 549      * way around) on OpenFlow Table 100.
 550      *
 551      * @param actionIn the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 552      * @param neutronFloatingIP An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronFloatingIP} instance of NeutronFloatingIP object.
 553      */
 554     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 555                                              Action actionIn) {
 556         Preconditions.checkNotNull(neutronFloatingIP);
 557
 558         LOG.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 559                 neutronFloatingIP.getFixedIPAddress(),
 560                 neutronFloatingIP.getFloatingIPAddress(),
 561                 neutronFloatingIP.getFloatingNetworkUUID());
 562         if (!this.enabled) {
 563             return;
 564         }
 565
 566         Action action;
 567
 568         // Consider action to be delete if getFixedIPAddress is null
 569         //
 570         if (neutronFloatingIP.getFixedIPAddress() == null) {
 571             action = Action.DELETE;
 572         } else {
 573             action = actionIn;
 574         }
 575
 576         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 577
 578         if (action != Action.DELETE) {
 579             // must be first, as it updates floatIpDataMapCache
 580             programFlowsForFloatingIPArpAdd(neutronFloatingIP);
 581
 582             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 583             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 584         } else {
 585             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 586             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 587
 588             // must be last, as it updates floatIpDataMapCache
 589             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID());
 590         }
 591     }
 592
 593     /**
 594      * This method performs creation or deletion of in-bound rules into Table 30 for a existing available floating
 595      * ip, otherwise for newer one.
 596      */
 597     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 598         Preconditions.checkNotNull(neutronFloatingIP);
 599
 600         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 601         if (fid == null) {
 602             LOG.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 603                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 604             return;
 605         }
 606         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 607                                       action);
 608     }
 609
 610     /**
 611      * This method performs creation or deletion of out-bound rules into Table 100 for a existing available floating
 612      * ip, otherwise for newer one.
 613      */
 614     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 615         Preconditions.checkNotNull(neutronFloatingIP);
 616
 617         final FloatIpData fid = getFloatingIPWithMetadata(neutronFloatingIP.getID());
 618         if (fid == null) {
 619             LOG.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 620                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 621             return;
 622         }
 623         programOutboundIpRewriteStage1(fid, action);
 624     }
 625
 626     private void flushExistingIpRewrite() {
 627         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 628             programOutboundIpRewriteStage1(fid, Action.DELETE);
 629         }
 630     }
 631
 632     private void rebuildExistingIpRewrite() {
 633         for (FloatIpData fid : getAllFloatingIPsWithMetadata()) {
 634             programOutboundIpRewriteStage1(fid, Action.ADD);
 635         }
 636     }
 637
 638     /**
 639      * This method creates ARP response rules into OpenFlow Table 30 for a given floating ip. In order to connect
 640      * to br-ex from br-int, a patch-port is used. Thus, the patch-port will be responsible to respond the ARP
 641      * requests.
 642      */
 643     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 644         Preconditions.checkNotNull(neutronFloatingIP);
 645         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 646         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 647
 648         // find bridge Node where floating ip is configured by looking up cache for its port
 649         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 650         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 651         final Pair<Long, Uuid> nodeIfPair = this.getDpIdOfNeutronPort(neutronTenantPortUuid);
 652         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 653         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 654         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 655
 656         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 657         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 658                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 659         final String providerSegmentationId = tenantNeutronNetwork != null ?
 660                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 661         final String neutronRouterMac = tenantNeutronNetwork != null ?
 662                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 663
 664         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 665                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 666                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 667                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 668             LOG.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 669                     fixedIpAddress,
 670                     floatingIpAddress,
 671                     neutronPortForFloatIp,
 672                     neutronTenantPortUuid,
 673                     providerSegmentationId,
 674                     floatingIpMac,
 675                     neutronRouterMac);
 676             return;
 677         }
 678
 679         // get ofport for patch port in br-int
 680         final Long dpId = nodeIfPair.getLeft();
 681         final Long ofPort = findOFPortForExtPatch(dpId);
 682         if (ofPort == null) {
 683             LOG.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 684                     dpId);
 685             return;
 686         }
 687
 688         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 689         //
 690         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 691                 Action.ADD)) {
 692             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 693                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 694             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 695             LOG.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 696                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 697                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 698         }
 699     }
 700
 701     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 702         final FloatIpData floatIpData = getFloatingIPWithMetadata(neutronFloatingIPUuid);
 703         if (floatIpData == null) {
 704             LOG.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 705             return;
 706         }
 707
 708         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 709                 floatIpData.floatingIpAddress, Action.DELETE)) {
 710             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 711             LOG.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 712                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 713         }
 714     }
 715
 716     private NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 717         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 718             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 719                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 720                 return neutronPort;
 721             }
 722         }
 723         return null;
 724     }
 725
 726     private Long findOFPortForExtPatch(Long dpId) {
 727         final String brInt = configurationService.getIntegrationBridgeName();
 728         final String brExt = configurationService.getExternalBridgeName();
 729         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 730
 731         Preconditions.checkNotNull(dpId);
 732         Preconditions.checkNotNull(portNameInt);
 733
 734         final long dpidPrimitive = dpId;
 735         for (Node node : nodeCacheManager.getBridgeNodes()) {
 736             if (dpidPrimitive == southbound.getDataPathId(node)) {
 737                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 738                         southbound.getTerminationPointOfBridge(node, portNameInt);
 739                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 740             }
 741         }
 742         return null;
 743     }
 744
 745     /**
 746      * Process the event.
 747      *
 748      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 749      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronFloatingIP object.
 750      */
 751     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 752         LOG.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 753     }
 754
 755     //
 756     // Callbacks from OVSDB's southbound handler
 757     //
 758     /**
 759      * Process the event.
 760      *
 761      * @param bridgeNode An instance of Node object.
 762      * @param intf An {@link org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105
 763      * .OvsdbTerminationPointAugmentation} instance of OvsdbTerminationPointAugmentation object.
 764      * @param neutronNetwork An {@link org.opendaylight.ovsdb.openstack.netvirt.translator.NeutronNetwork} instance of NeutronNetwork
 765      * object.
 766      * @param action the {@link org.opendaylight.ovsdb.openstack.netvirt.api.Action} action to be handled.
 767      */
 768     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 769                                      final NeutronNetwork neutronNetwork, Action action) {
 770         LOG.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 771                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 772         if (!this.enabled) {
 773             return;
 774         }
 775
 776         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 777         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 778         final Uuid interfaceUuid = intf.getInterfaceUuid();
 779
 780         LOG.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 781                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 782
 783         if (neutronPort != null) {
 784             final String neutronPortUuid = neutronPort.getPortUUID();
 785
 786             if (action != Action.DELETE && dpId != null && interfaceUuid != null) {
 787                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 788             }
 789
 790             handleNeutronPortEvent(neutronPort, action);
 791         }
 792
 793         if (action == Action.DELETE && interfaceUuid != null) {
 794             handleInterfaceEventDelete(intf, dpId);
 795         }
 796     }
 797
 798     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 799         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 800         LOG.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 801                 neutronPortUuid, dpId, interfaceUuid.getValue());
 802     }
 803
 804     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 805         // Remove entry from neutronPortToDpIdCache based on interface uuid
 806         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 807             final String currPortUuid = entry.getKey();
 808             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 809                 LOG.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 810                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 811                 neutronPortToDpIdCache.remove(currPortUuid);
 812                 break;
 813             }
 814         }
 815     }
 816
 817     //
 818     // Internal helpers
 819     //
 820     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 821
 822         final String networkUUID = neutronPort.getNetworkUUID();
 823         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 824
 825         // If there is no router interface handling the networkUUID, we are done
 826         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 827             return;
 828         }
 829
 830         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 831         // router interface are handled via handleNeutronRouterInterfaceEvent.
 832         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 833             return;
 834         }
 835
 836         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 837         final String providerSegmentationId = neutronNetwork != null ?
 838                                               neutronNetwork.getProviderSegmentationID() : null;
 839         final String tenantMac = neutronPort.getMacAddress();
 840
 841         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 842             tenantMac == null || tenantMac.isEmpty()) {
 843             // done: go no further w/out all the info needed...
 844             return;
 845         }
 846
 847         final Action action = isDelete ? Action.DELETE : Action.ADD;
 848         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 849         if (nodes.isEmpty()) {
 850             LOG.trace("updateL3ForNeutronPort has no nodes to work with");
 851         }
 852         for (Node node : nodes) {
 853             final Long dpid = getDpidForIntegrationBridge(node);
 854             if (dpid == null) {
 855                 continue;
 856             }
 857             if (neutronPort.getFixedIPs() == null) {
 858                 continue;
 859             }
 860             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 861                 final String tenantIpStr = neutronIP.getIpAddress();
 862                 if (tenantIpStr.isEmpty()) {
 863                     continue;
 864                 }
 865
 866                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 867                 // still needed when routing to subnets non-local to node (bug 2076).
 868                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 869
 870                 // Configure distributed ARP responder
 871                 if (flgDistributedARPEnabled) {
 872                     // Arp rule is only needed when segmentation exists in the given node (bug 4752).
 873                     boolean arpNeeded = tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId);
 874                     final Action actionForNode = arpNeeded ? action : Action.DELETE;
 875                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, actionForNode);
 876                 }
 877             }
 878         }
 879     }
 880
 881     private void processSecurityGroupUpdate(NeutronPort neutronPort) {
 882         LOG.trace("processSecurityGroupUpdate:" + neutronPort);
 883         /**
 884          * Get updated data and original data for the the changed. Identify the security groups that got
 885          * added and removed and call the appropriate providers for updating the flows.
 886          */
 887         try {
 888             NeutronPort originalPort = neutronPort.getOriginalPort();
 889             List<NeutronSecurityGroup> addedGroup = getsecurityGroupChanged(neutronPort,
 890                                                                             neutronPort.getOriginalPort());
 891             List<NeutronSecurityGroup> deletedGroup = getsecurityGroupChanged(neutronPort.getOriginalPort(),
 892                                                                               neutronPort);
 893
 894             if (null != addedGroup && !addedGroup.isEmpty()) {
 895                 securityServicesManager.syncSecurityGroup(neutronPort,addedGroup,true);
 896             }
 897             if (null != deletedGroup && !deletedGroup.isEmpty()) {
 898                 securityServicesManager.syncSecurityGroup(neutronPort,deletedGroup,false);
 899             }
 900
 901         } catch (Exception e) {
 902             LOG.error("Exception in processSecurityGroupUpdate", e);
 903         }
 904     }
 905
 906     private List<NeutronSecurityGroup> getsecurityGroupChanged(NeutronPort port1, NeutronPort port2) {
 907         LOG.trace("getsecurityGroupChanged:" + "Port1:" + port1 + "Port2" + port2);
 908         if (port1 == null) {
 909             return null;
 910         }
 911         List<NeutronSecurityGroup> list1 = new ArrayList<>(port1.getSecurityGroups());
 912         if (port2 == null) {
 913             return list1;
 914         }
 915         List<NeutronSecurityGroup> list2 = new ArrayList<>(port2.getSecurityGroups());
 916         for (Iterator<NeutronSecurityGroup> iterator = list1.iterator(); iterator.hasNext();) {
 917             NeutronSecurityGroup securityGroup1 = iterator.next();
 918             for (NeutronSecurityGroup securityGroup2 :list2) {
 919                 if (securityGroup1.getID().equals(securityGroup2.getID())) {
 920                     iterator.remove();
 921                 }
 922             }
 923         }
 924         return list1;
 925     }
 926
 927     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 928                                            String macAddress, String ipStr,
 929                                            Action actionForNode) {
 930         if (actionForNode == Action.DELETE) {
 931             LOG.trace("Deleting Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 932                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 933         }
 934         if (actionForNode == Action.ADD) {
 935             LOG.trace("Adding Flow : programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {}",
 936                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 937         }
 938
 939         this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 940                                                        macAddress, ipStr, actionForNode);
 941     }
 942
 943     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 944                                              String macAddress,
 945                                              String address,
 946                                              Action actionForNode) {
 947         Status status;
 948         try {
 949             InetAddress inetAddress = InetAddress.getByName(address);
 950             status = l3ForwardingProvider == null ?
 951                      new Status(StatusCode.SUCCESS) :
 952                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 953                                                                       inetAddress, macAddress, actionForNode);
 954         } catch (UnknownHostException e) {
 955             status = new Status(StatusCode.BADREQUEST);
 956         }
 957
 958         if (status.isSuccess()) {
 959             LOG.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 960                          l3ForwardingProvider == null ? "skipped" : "programmed",
 961                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 962         } else {
 963             LOG.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 964                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 965         }
 966         return status;
 967     }
 968
 969     // --
 970
 971     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 972                                                        Boolean isDelete) {
 973         Preconditions.checkNotNull(destNeutronRouterInterface);
 974
 975         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 976         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 977         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 978         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 979         final NeutronNetwork neutronNetwork = subnet != null ?
 980                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 981         final String destinationSegmentationId = neutronNetwork != null ?
 982                                                  neutronNetwork.getProviderSegmentationID() : null;
 983         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 984         final String cidr = subnet != null ? subnet.getCidr() : null;
 985         final int mask = getMaskLenFromCidr(cidr);
 986
 987         LOG.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 988                      destNeutronRouterInterface, isDelete);
 989
 990         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 991         // the local cache
 992         if (neutronNetwork != null) {
 993             if (macAddress == null || macAddress.isEmpty()) {
 994                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 995             }
 996             if (ipList == null || ipList.isEmpty()) {
 997                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 998             }
 999         }
1000
1001         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
1002             cidr == null || cidr.isEmpty() ||
1003             macAddress == null || macAddress.isEmpty() ||
1004             ipList == null || ipList.isEmpty()) {
1005             LOG.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
1006                          destinationSegmentationId, cidr, macAddress, ipList);
1007             // done: go no further w/out all the info needed...
1008             return;
1009         }
1010
1011         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
1012
1013         // Keep cache for finding router's mac from network uuid -- add
1014         //
1015         if (! isDelete) {
1016             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
1017             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
1018             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
1019         }
1020
1021         List<Node> nodes = nodeCacheManager.getBridgeNodes();
1022         if (nodes.isEmpty()) {
1023             LOG.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
1024         }
1025         for (Node node : nodes) {
1026             final Long dpid = getDpidForIntegrationBridge(node);
1027             if (dpid == null) {
1028                 continue;
1029             }
1030
1031             for (Neutron_IPs neutronIP : ipList) {
1032                 final String ipStr = neutronIP.getIpAddress();
1033                 if (ipStr.isEmpty()) {
1034                     LOG.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
1035                             node.getNodeId().getValue(), ipStr);
1036                     continue;
1037                 }
1038
1039                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
1040                 //
1041                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
1042                     programFlowsForNeutronRouterInterfacePair(node, dpid,
1043                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
1044                                                               neutronNetwork, destinationSegmentationId,
1045                                                               macAddress, ipStr, mask, actionForNode,
1046                                                               true /*isReflexsive*/);
1047                 }
1048
1049                 if (! isExternal) {
1050                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
1051                             actionForNode);
1052                 }
1053                 // Enable ARP responder by default, because router interface needs to be responded always.
1054                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1055                 programIcmpEcho(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
1056             }
1057
1058             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
1059             // for the external neutron networks.
1060             //
1061             {
1062                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
1063                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
1064             }
1065         }
1066
1067         // Keep cache for finding router's mac from network uuid -- NOTE: remove is done later, via cleanupRouterCache()
1068     }
1069
1070     private void programFlowForNetworkFromExternal(final Node node,
1071                                                    final Long dpid,
1072                                                    final String destinationSegmentationId,
1073                                                    final String dstMacAddress,
1074                                                    final String destIpStr,
1075                                                    final int destMask,
1076                                                    final Action actionForNode) {
1077         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
1078                 dstMacAddress, destIpStr, destMask, actionForNode);
1079     }
1080
1081     private void programFlowsForNeutronRouterInterfacePair(final Node node,
1082                                                            final Long dpid,
1083                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
1084                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
1085                                                            final NeutronNetwork dstNeutronNetwork,
1086                                                            final String destinationSegmentationId,
1087                                                            final String dstMacAddress,
1088                                                            final String destIpStr,
1089                                                            final int destMask,
1090                                                            final Action actionForNode,
1091                                                            Boolean isReflexsive) {
1092         Preconditions.checkNotNull(srcNeutronRouterInterface);
1093         Preconditions.checkNotNull(dstNeutronRouterInterface);
1094
1095         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
1096         if (sourceSubnetId == null) {
1097             LOG.error("Could not get provider Subnet ID from router interface {}",
1098                          srcNeutronRouterInterface.getID());
1099             return;
1100         }
1101
1102         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
1103         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
1104         if (sourceNetworkId == null) {
1105             LOG.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
1106             return;
1107         }
1108
1109         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
1110         if (sourceNetwork == null) {
1111             LOG.error("Could not get provider Network for Network ID {}", sourceNetworkId);
1112             return;
1113         }
1114
1115         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
1116             // Isolate subnets from different tenants within the same router
1117             return;
1118         }
1119         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
1120         if (sourceSegmentationId == null) {
1121             LOG.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
1122             return;
1123         }
1124         if (sourceSegmentationId.equals(destinationSegmentationId)) {
1125             // Skip 'self'
1126             return;
1127         }
1128
1129         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
1130                                      dstMacAddress, destIpStr, destMask, actionForNode);
1131
1132         // Flip roles src->dst; dst->src
1133         if (isReflexsive) {
1134             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
1135             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
1136             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
1137             final String cidr2 = sourceSubnet.getCidr();
1138             final int mask2 = getMaskLenFromCidr(cidr2);
1139
1140             if (cidr2 == null || cidr2.isEmpty() ||
1141                 macAddress2 == null || macAddress2.isEmpty() ||
1142                 ipList2 == null || ipList2.isEmpty()) {
1143                 LOG.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
1144                              sourceSegmentationId, cidr2, macAddress2, ipList2);
1145                 // done: go no further w/out all the info needed...
1146                 return;
1147             }
1148
1149             for (Neutron_IPs neutronIP2 : ipList2) {
1150                 final String ipStr2 = neutronIP2.getIpAddress();
1151                 if (ipStr2.isEmpty()) {
1152                     continue;
1153                 }
1154                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
1155                                                           srcNeutronRouterInterface,
1156                                                           sourceNetwork, sourceSegmentationId,
1157                                                           macAddress2, ipStr2, mask2, actionForNode,
1158                                                           false /*isReflexsive*/);
1159             }
1160         }
1161     }
1162
1163     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
1164                                               String destinationSegmentationId,
1165                                               String macAddress, String ipStr, int mask,
1166                                               Action actionForNode) {
1167         if (actionForNode == Action.DELETE) {
1168             LOG.trace("Deleting Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1169                          " action {}",
1170                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1171                          macAddress, ipStr, mask, actionForNode);
1172         }
1173         if (actionForNode == Action.ADD) {
1174             LOG.trace("Adding Flow : programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
1175                          " action {}",
1176                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1177                          macAddress, ipStr, mask, actionForNode);
1178         }
1179
1180         this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
1181                                                           macAddress, ipStr, mask, actionForNode);
1182     }
1183
1184     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
1185                                                 String destinationSegmentationId,
1186                                                 String macAddress,
1187                                                 String address, int mask,
1188                                                 Action actionForNode) {
1189         Status status;
1190         try {
1191             InetAddress inetAddress = InetAddress.getByName(address);
1192             status = routingProvider == null ?
1193                      new Status(StatusCode.SUCCESS) :
1194                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
1195                                                             macAddress, inetAddress, mask, actionForNode);
1196         } catch (UnknownHostException e) {
1197             status = new Status(StatusCode.BADREQUEST);
1198         }
1199
1200         if (status.isSuccess()) {
1201             LOG.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
1202                          routingProvider == null ? "skipped" : "programmed",
1203                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1204                          actionForNode);
1205         } else {
1206             LOG.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
1207                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
1208                          actionForNode, status);
1209         }
1210         return status;
1211     }
1212
1213     private boolean programIcmpEcho(Long dpid, String segOrOfPort,
1214                                            String macAddress, String ipStr,
1215                                            Action action) {
1216         if (action == Action.DELETE ) {
1217             LOG.trace("Deleting Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1218                     dpid, segOrOfPort, macAddress, ipStr, action);
1219         }
1220         if (action == Action.ADD) {
1221             LOG.trace("Adding Flow : programIcmpEcho dpid {} segOrOfPort {} mac {} ip {} action {}",
1222                     dpid, segOrOfPort, macAddress, ipStr, action);
1223         }
1224
1225         Status status = new Status(StatusCode.UNSUPPORTED);
1226         if (icmpEchoProvider != null){
1227             try {
1228                 InetAddress inetAddress = InetAddress.getByName(ipStr);
1229                 status = icmpEchoProvider.programIcmpEchoEntry(dpid, segOrOfPort,
1230                                                 macAddress, inetAddress, action);
1231             } catch (UnknownHostException e) {
1232                 status = new Status(StatusCode.BADREQUEST);
1233             }
1234         }
1235
1236         if (status.isSuccess()) {
1237             LOG.debug("programIcmpEcho {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1238                     icmpEchoProvider == null ? "skipped" : "programmed",
1239                     macAddress, ipStr, dpid, segOrOfPort, action);
1240         } else {
1241             LOG.error("programIcmpEcho failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1242                     macAddress, ipStr, dpid, segOrOfPort, action, status);
1243         }
1244
1245         return status.isSuccess();
1246     }
1247
1248     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
1249                                            String macAddress, String ipStr,
1250                                            Action action) {
1251         if (action == Action.DELETE ) {
1252             LOG.trace("Deleting Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1253                     dpid, segOrOfPort, macAddress, ipStr, action);
1254         }
1255         if (action == Action.ADD) {
1256             LOG.trace("Adding Flow : programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {}",
1257                     dpid, segOrOfPort, macAddress, ipStr, action);
1258         }
1259
1260         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
1261         return status.isSuccess();
1262     }
1263
1264     private Status programStaticArpStage2(Long dpid,
1265                                           String segOrOfPort,
1266                                           String macAddress,
1267                                           String address,
1268                                           Action action) {
1269         Status status;
1270         try {
1271             InetAddress inetAddress = InetAddress.getByName(address);
1272             status = arpProvider == null ?
1273                      new Status(StatusCode.SUCCESS) :
1274                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
1275                                                        macAddress, inetAddress, action);
1276         } catch (UnknownHostException e) {
1277             status = new Status(StatusCode.BADREQUEST);
1278         }
1279
1280         if (status.isSuccess()) {
1281             LOG.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
1282                          arpProvider == null ? "skipped" : "programmed",
1283                          macAddress, address, dpid, segOrOfPort, action);
1284         } else {
1285             LOG.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
1286                          macAddress, address, dpid, segOrOfPort, action, status);
1287         }
1288         return status;
1289     }
1290
1291     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
1292                                                   String matchAddress, String rewriteAddress,
1293                                                   Action action) {
1294         if (action == Action.DELETE ) {
1295             LOG.trace("Deleting Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1296                     " action {}",
1297                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1298         }
1299         if (action == Action.ADD ) {
1300             LOG.trace("Adding Flow : programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
1301                     " action {}",
1302                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1303         }
1304
1305         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
1306                 rewriteAddress, action);
1307         return status.isSuccess();
1308     }
1309
1310     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
1311                                                  String matchAddress, String rewriteAddress,
1312                                                  Action action) {
1313         Status status;
1314         try {
1315             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
1316             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
1317             status = inboundNatProvider == null ?
1318                     new Status(StatusCode.SUCCESS) :
1319                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
1320                             inetMatchAddress, inetRewriteAddress,
1321                             action);
1322         } catch (UnknownHostException e) {
1323             status = new Status(StatusCode.BADREQUEST);
1324         }
1325
1326         if (status.isSuccess()) {
1327             final boolean isSkipped = inboundNatProvider == null;
1328             LOG.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
1329                     isSkipped ? "skipped" : "programmed",
1330                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
1331         } else {
1332             LOG.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
1333                          " status:{}",
1334                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
1335                     status);
1336         }
1337         return status;
1338     }
1339
1340     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1341                                                  Action actionForRewriteExclusion) {
1342         if (actionForRewriteExclusion == Action.DELETE ) {
1343             LOG.trace("Deleting Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1344                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1345         }
1346         if (actionForRewriteExclusion == Action.ADD) {
1347             LOG.trace("Adding Flow : programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {}",
1348                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1349         }
1350
1351         this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,actionForRewriteExclusion);
1352     }
1353
1354     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1355                                                    Action actionForNode) {
1356         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1357                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1358
1359         if (status.isSuccess()) {
1360             final boolean isSkipped = outboundNatProvider == null;
1361             LOG.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1362                          isSkipped ? "skipped" : "programmed",
1363                          cidr, node.getNodeId().getValue(), actionForNode);
1364         } else {
1365             LOG.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1366                          cidr, node.getNodeId().getValue(), actionForNode, status);
1367         }
1368         return status;
1369     }
1370
1371     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1372
1373         if (action == Action.DELETE) {
1374             LOG.trace("Deleting Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} ",
1375                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1376         }
1377         if (action == Action.ADD) {
1378             LOG.trace("Adding Flow : programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " ,
1379                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1380         }
1381
1382         this.programOutboundIpRewriteStage2(fid, action);
1383     }
1384
1385     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1386         Status status;
1387         try {
1388             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1389             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1390             status = outboundNatProvider == null ?
1391                     new Status(StatusCode.SUCCESS) :
1392                     outboundNatProvider.programIpRewriteRule(
1393                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1394                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1395         } catch (UnknownHostException e) {
1396             status = new Status(StatusCode.BADREQUEST);
1397         }
1398
1399         if (status.isSuccess()) {
1400             final boolean isSkipped = outboundNatProvider == null;
1401             LOG.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1402                             " action {}",
1403                          isSkipped ? "skipped" : "programmed",
1404                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1405         } else {
1406             LOG.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1407                          " action {} status:{}",
1408                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1409         }
1410         return status;
1411     }
1412
1413     private int getMaskLenFromCidr(String cidr) {
1414         if (cidr == null) {
1415             return 0;
1416         }
1417         String[] splits = cidr.split("/");
1418         if (splits.length != 2) {
1419             return 0;
1420         }
1421
1422         int result;
1423         try {
1424             result = Integer.parseInt(splits[1].trim());
1425         } catch (NumberFormatException nfe) {
1426             result = 0;
1427         }
1428         return result;
1429     }
1430
1431     private Long getDpidForIntegrationBridge(Node node) {
1432         // Check if node is integration bridge; and only then return its dpid
1433         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1434             return southbound.getDataPathId(node);
1435         }
1436         return null;
1437     }
1438
1439     private Long getDpidForExternalBridge(Node node) {
1440         // Check if node is external bridge; and only then return its dpid
1441         if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1442             return southbound.getDataPathId(node);
1443         }
1444         return null;
1445     }
1446
1447     private Node getExternalBridgeNode(){
1448         //Pickup the first node that has external bridge (br-ex).
1449         //NOTE: We are assuming that all the br-ex are serving one external network and gateway ip of
1450         //the external network is reachable from every br-ex
1451         // TODO: Consider other deployment scenario, and thing of better solution.
1452         List<Node> allBridges = nodeCacheManager.getBridgeNodes();
1453         for(Node node : allBridges){
1454             if (southbound.getBridge(node, configurationService.getExternalBridgeName()) != null) {
1455                 return node;
1456             }
1457         }
1458         return null;
1459     }
1460
1461     private NeutronSubnet getExternalNetworkSubnet(NeutronPort gatewayPort){
1462         if (gatewayPort.getFixedIPs() == null) {
1463             return null;
1464         }
1465         for (Neutron_IPs neutronIPs : gatewayPort.getFixedIPs()) {
1466             String subnetUUID = neutronIPs.getSubnetUUID();
1467             NeutronSubnet extSubnet = neutronSubnetCache.getSubnet(subnetUUID);
1468             if (extSubnet != null && extSubnet.getGatewayIP() != null) {
1469                 return extSubnet;
1470             }
1471             if (extSubnet == null) {
1472                 // TODO: when subnet is created, try again.
1473                 LOG.debug("subnet {} in not found", subnetUUID);
1474              }
1475         }
1476         return null;
1477     }
1478
1479      private void cleanupRouterCache(final NeutronRouter_Interface neutronRouterInterface) {
1480          /*
1481           *  Fix for 4277
1482           *  Remove the router cache only after deleting the neutron
1483           *  port l3 flows.
1484           */
1485          final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
1486
1487          if (neutronPort != null) {
1488              networkIdToRouterMacCache.remove(neutronPort.getNetworkUUID());
1489              networkIdToRouterIpListCache.remove(neutronPort.getNetworkUUID());
1490              subnetIdToRouterInterfaceCache.remove(neutronRouterInterface.getSubnetUUID());
1491          }
1492      }
1493
1494     private void triggerGatewayMacResolver(final NeutronPort gatewayPort){
1495
1496         Preconditions.checkNotNull(gatewayPort);
1497         NeutronNetwork externalNetwork = neutronNetworkCache.getNetwork(gatewayPort.getNetworkUUID());
1498
1499         if(externalNetwork != null){
1500             if(externalNetwork.isRouterExternal()){
1501                 final NeutronSubnet externalSubnet = getExternalNetworkSubnet(gatewayPort);
1502
1503                 // TODO: address IPv6 case.
1504                 if (externalSubnet != null &&
1505                         externalSubnet.getIpVersion() == 4 &&
1506                         gatewayPort.getFixedIPs() != null) {
1507                     LOG.info("Trigger MAC resolution for gateway ip {}", externalSubnet.getGatewayIP());
1508
1509                     gatewayMacResolver.resolveMacAddress(
1510                             this, /* gatewayMacResolverListener */
1511                             null, /* externalNetworkBridgeDpid */
1512                             true, /* refreshExternalNetworkBridgeDpidIfNeeded */
1513                             new Ipv4Address(externalSubnet.getGatewayIP()),
1514                             new Ipv4Address(gatewayPort.getFixedIPs().get(0).getIpAddress()),
1515                             new MacAddress(gatewayPort.getMacAddress()),
1516                             true /* periodicRefresh */);
1517                 } else {
1518                     LOG.warn("No gateway IP address found for external network {}", externalNetwork);
1519                 }
1520             }
1521         }else{
1522             LOG.warn("Neutron network not found for router interface {}", gatewayPort);
1523         }
1524     }
1525
1526     /**
1527      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1528      *
1529      * @param ofPort the OF port number
1530      * @return the string with encoded OF port (example format "OFPort|999")
1531      */
1532     public static String encodeExcplicitOFPort(Long ofPort) {
1533         return "OFPort|" + ofPort.toString();
1534     }
1535
1536     @Override
1537     public void setDependencies(ServiceReference serviceReference) {
1538         eventDispatcher =
1539                 (EventDispatcher) ServiceHelper.getGlobalInstance(EventDispatcher.class, this);
1540         eventDispatcher.eventHandlerAdded(serviceReference, this);
1541         tenantNetworkManager =
1542                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1543         configurationService =
1544                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1545         arpProvider =
1546                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1547         inboundNatProvider =
1548                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1549         outboundNatProvider =
1550                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1551         routingProvider =
1552                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1553         l3ForwardingProvider =
1554                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1555         nodeCacheManager =
1556                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1557         southbound =
1558                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1559         gatewayMacResolver =
1560                 (GatewayMacResolver) ServiceHelper.getGlobalInstance(GatewayMacResolver.class, this);
1561         securityServicesManager =
1562                 (SecurityServicesManager) ServiceHelper.getGlobalInstance(SecurityServicesManager.class, this);
1563
1564         initL3AdapterMembers();
1565     }
1566
1567     @Override
1568     public void setDependencies(Object impl) {
1569         if (impl instanceof INeutronNetworkCRUD) {
1570             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1571         } else if (impl instanceof INeutronPortCRUD) {
1572             neutronPortCache = (INeutronPortCRUD)impl;
1573         } else if (impl instanceof INeutronSubnetCRUD) {
1574             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1575         } else if (impl instanceof INeutronFloatingIPCRUD) {
1576             neutronFloatingIpCache = (INeutronFloatingIPCRUD)impl;
1577         } else if (impl instanceof ArpProvider) {
1578             arpProvider = (ArpProvider)impl;
1579         } else if (impl instanceof InboundNatProvider) {
1580             inboundNatProvider = (InboundNatProvider)impl;
1581         } else if (impl instanceof OutboundNatProvider) {
1582             outboundNatProvider = (OutboundNatProvider)impl;
1583         } else if (impl instanceof RoutingProvider) {
1584             routingProvider = (RoutingProvider)impl;
1585         } else if (impl instanceof L3ForwardingProvider) {
1586             l3ForwardingProvider = (L3ForwardingProvider)impl;
1587         }else if (impl instanceof GatewayMacResolver) {
1588             gatewayMacResolver = (GatewayMacResolver)impl;
1589         }else if (impl instanceof IcmpEchoProvider) {
1590             icmpEchoProvider = (IcmpEchoProvider)impl;
1591         }
1592
1593         populateL3ForwardingCaches();
1594     }
1595 }