openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  29 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  30
  31 import com.google.common.base.Preconditions;
  32 import org.osgi.framework.BundleContext;
  33 import org.osgi.framework.ServiceReference;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.net.InetAddress;
  38 import java.net.UnknownHostException;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.ArrayList;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45
  46 /**
  47  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  48  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  49  * as well as the multi-tenant router forwarding provider.
  50  */
  51 public class NeutronL3Adapter implements ConfigInterface {
  52     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  53
  54     // The implementation for each of these services is resolved by the OSGi Service Manager
  55     private volatile ConfigurationService configurationService;
  56     private volatile TenantNetworkManager tenantNetworkManager;
  57     private volatile NodeCacheManager nodeCacheManager;
  58     private volatile INeutronNetworkCRUD neutronNetworkCache;
  59     private volatile INeutronSubnetCRUD neutronSubnetCache;
  60     private volatile INeutronPortCRUD neutronPortCache;
  61     private volatile L3ForwardingProvider l3ForwardingProvider;
  62     private volatile InboundNatProvider inboundNatProvider;
  63     private volatile OutboundNatProvider outboundNatProvider;
  64     private volatile ArpProvider arpProvider;
  65     private volatile RoutingProvider routingProvider;
  66
  67     private class FloatIpData {
  68         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  69                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  70             this.dpid = dpid;
  71             this.ofPort = ofPort;
  72             this.segId = segId;
  73             this.macAddress = macAddress;
  74             this.floatingIpAddress = floatingIpAddress;
  75             this.fixedIpAddress = fixedIpAddress;
  76             this.neutronRouterMac = neutronRouterMac;
  77         }
  78
  79         public final Long dpid;          // br-int of node where floating ip is associated with tenant port
  80         public final Long ofPort;        // patch port in br-int used to reach br-ex
  81         public final String segId;       // segmentation id of the net where fixed ip is instantiated
  82         public final String macAddress;  // mac address assigned to neutron port of floating ip
  83         public final String floatingIpAddress;
  84         public final String fixedIpAddress;  // ip address given to tenant vm
  85         public final String neutronRouterMac;
  86     }
  87
  88     private Set<String> inboundIpRewriteCache;
  89     private Set<String> outboundIpRewriteCache;
  90     private Set<String> outboundIpRewriteExclusionCache;
  91     private Set<String> routerInterfacesCache;
  92     private Set<String> staticArpEntryCache;
  93     private Set<String> l3ForwardingCache;
  94     private Map<String, String> networkIdToRouterMacCache;
  95     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
  96     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  97     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
  98     private Map<String, FloatIpData> floatIpDataMapCache;
  99     private String externalRouterMac;
 100     private Boolean enabled = false;
 101     private Boolean flgDistributedARPEnabled = true;
 102     private Southbound southbound;
 103
 104     final static String OWNER_ROUTER_INTERFACE = "network:router_interface";
 105     final static String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 106     final static String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 107     final static String OWNER_FLOATING_IP = "network:floatingip";
 108     final static String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 109
 110     public NeutronL3Adapter() {
 111         logger.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 112     }
 113
 114     private void initL3AdapterMembers() {
 115         Preconditions.checkNotNull(configurationService);
 116
 117         if (configurationService.isL3ForwardingEnabled()) {
 118             this.inboundIpRewriteCache = new HashSet<>();
 119             this.outboundIpRewriteCache = new HashSet<>();
 120             this.outboundIpRewriteExclusionCache = new HashSet<>();
 121             this.routerInterfacesCache = new HashSet<>();
 122             this.staticArpEntryCache = new HashSet<>();
 123             this.l3ForwardingCache = new HashSet<>();
 124             this.networkIdToRouterMacCache = new HashMap<>();
 125             this.networkIdToRouterIpListCache = new HashMap<>();
 126             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 127             this.neutronPortToDpIdCache = new HashMap<>();
 128             this.floatIpDataMapCache = new HashMap<>();
 129
 130             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 131             if (this.externalRouterMac == null) {
 132                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 133             }
 134
 135             this.enabled = true;
 136             logger.info("OVSDB L3 forwarding is enabled");
 137             if (configurationService.isDistributedArpDisabled()) {
 138                 this.flgDistributedARPEnabled = false;
 139                 logger.info("Distributed ARP responder is disabled");
 140             } else {
 141                 logger.debug("Distributed ARP responder is enabled");
 142             }
 143         } else {
 144             logger.debug("OVSDB L3 forwarding is disabled");
 145         }
 146     }
 147
 148     //
 149     // Callbacks from OVSDB's northbound handlers
 150     //
 151
 152     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 153         Preconditions.checkNotNull(externalRouterMacUpdate);
 154
 155         flushExistingIpRewrite();
 156         this.externalRouterMac = externalRouterMacUpdate;
 157         rebuildExistingIpRewrite();
 158     }
 159
 160     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 161         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 162         if (!this.enabled)
 163             return;
 164     }
 165
 166     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 167         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 168         if (!this.enabled)
 169             return;
 170
 171         final boolean isDelete = action == Action.DELETE;
 172
 173         // Treat the port event as a router interface event if the port belongs to router. This is a
 174         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 175         //
 176         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 177             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 178             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 179                 NeutronRouter_Interface neutronRouterInterface =
 180                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 181                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 182                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 183
 184                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 185             }
 186         } else {
 187             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 188             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 189             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 190             // there.
 191             //
 192             if (!isDelete) {
 193                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 194                     NeutronRouter_Interface neutronRouterInterface =
 195                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 196                     if (neutronRouterInterface != null) {
 197                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 198                     }
 199                 }
 200             }
 201             this.updateL3ForNeutronPort(neutronPort, isDelete);
 202         }
 203     }
 204
 205     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 206         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 207         if (!this.enabled)
 208             return;
 209     }
 210
 211     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 212                                                   final NeutronRouter_Interface neutronRouterInterface,
 213                                                   Action action) {
 214         logger.debug("Router interface {} got event {}. Subnet {}",
 215                      neutronRouterInterface.getPortUUID(),
 216                      action,
 217                      neutronRouterInterface.getSubnetUUID());
 218         if (!this.enabled)
 219             return;
 220
 221         final boolean isDelete = action == Action.DELETE;
 222
 223         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 224
 225         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 226         // see if they are affected by l3
 227         //
 228         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 229             boolean currPortShouldBeDeleted = false;
 230             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 231             if (isDelete) {
 232                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 233                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 234                         currPortShouldBeDeleted = true;
 235                         break;
 236                     }
 237                 }
 238             }
 239             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 240         }
 241     }
 242
 243     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 244                                              Action action) {
 245         Preconditions.checkNotNull(neutronFloatingIP);
 246
 247         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 248                 neutronFloatingIP.getFixedIPAddress(),
 249                 neutronFloatingIP.getFloatingIPAddress(),
 250                 neutronFloatingIP.getFloatingNetworkUUID());
 251         if (!this.enabled)
 252             return;
 253
 254         // Consider action to be delete if getFixedIPAddress is null
 255         //
 256         if (neutronFloatingIP.getFixedIPAddress() == null) {
 257             action = Action.DELETE;
 258         }
 259
 260         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 261
 262         if (action != Action.DELETE) {
 263             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 264
 265             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 266             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 267         } else {
 268             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 269             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 270
 271             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 272         }
 273     }
 274
 275     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 276         Preconditions.checkNotNull(neutronFloatingIP);
 277
 278         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 279         if (fid == null) {
 280             logger.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 281                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 282             return;
 283         }
 284         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 285                                       action);
 286     }
 287
 288     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 289         Preconditions.checkNotNull(neutronFloatingIP);
 290
 291         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 292         if (fid == null) {
 293             logger.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 294                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 295             return;
 296         }
 297         programOutboundIpRewriteStage1(fid, action);
 298     }
 299
 300     private void flushExistingIpRewrite() {
 301         for (FloatIpData fid : floatIpDataMapCache.values()) {
 302             programOutboundIpRewriteStage1(fid, Action.DELETE);
 303         }
 304     }
 305
 306     private void rebuildExistingIpRewrite() {
 307         for (FloatIpData fid : floatIpDataMapCache.values()) {
 308             programOutboundIpRewriteStage1(fid, Action.ADD);
 309         }
 310     }
 311
 312     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 313         Preconditions.checkNotNull(neutronFloatingIP);
 314         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 315         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 316
 317         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 318             logger.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 319                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 320             return;
 321         }
 322
 323         // find bridge Node where floating ip is configured by looking up cache for its port
 324         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 325         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 326         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 327         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 328         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 329         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 330
 331         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 332         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 333                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 334         final String providerSegmentationId = tenantNeutronNetwork != null ?
 335                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 336         final String neutronRouterMac = tenantNeutronNetwork != null ?
 337                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 338
 339         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 340                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 341                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 342                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 343             logger.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 344                     fixedIpAddress,
 345                     floatingIpAddress,
 346                     neutronPortForFloatIp,
 347                     neutronTenantPortUuid,
 348                     providerSegmentationId,
 349                     floatingIpMac,
 350                     neutronRouterMac);
 351             return;
 352         }
 353
 354         // get ofport for patch port in br-int
 355         final Long dpId = nodeIfPair.getLeft();
 356         final Long ofPort = findOFPortForExtPatch(dpId);
 357         if (ofPort == null) {
 358             logger.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 359                     dpId);
 360             return;
 361         }
 362
 363         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 364         //
 365         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 366                 Action.ADD)) {
 367             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 368                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 369             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 370             logger.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 371                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 372                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 373         }
 374     }
 375
 376     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 377         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 378         if (floatIpData == null) {
 379             logger.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 380             return;
 381         }
 382
 383         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 384                 floatIpData.floatingIpAddress, Action.DELETE)) {
 385             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 386             logger.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 387                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 388         }
 389     }
 390
 391     private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 392         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 393             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 394                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 395                 return neutronPort;
 396             }
 397         }
 398         return null;
 399     }
 400
 401     private final Long findOFPortForExtPatch(Long dpId) {
 402         final String brInt = configurationService.getIntegrationBridgeName();
 403         final String brExt = configurationService.getExternalBridgeName();
 404         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 405
 406         Preconditions.checkNotNull(dpId);
 407         Preconditions.checkNotNull(portNameInt);
 408
 409         final long dpidPrimitive = dpId.longValue();
 410         for (Node node : nodeCacheManager.getBridgeNodes()) {
 411             if (dpidPrimitive == southbound.getDataPathId(node)) {
 412                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 413                         southbound.getTerminationPointOfBridge(node, portNameInt);
 414                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 415             }
 416         }
 417         return null;
 418     }
 419
 420     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 421         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 422         if (!this.enabled)
 423             return;
 424     }
 425
 426     //
 427     // Callbacks from OVSDB's southbound handler
 428     //
 429     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 430                                      final NeutronNetwork neutronNetwork, Action action) {
 431         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 432                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 433         if (!this.enabled) {
 434             return;
 435         }
 436
 437         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 438         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 439         final Uuid interfaceUuid = intf.getInterfaceUuid();
 440
 441         logger.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 442                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 443
 444         if (neutronPort != null) {
 445             final String neutronPortUuid = neutronPort.getPortUUID();
 446
 447             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 448                     dpId != null && interfaceUuid != null) {
 449                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 450             }
 451
 452             handleNeutronPortEvent(neutronPort, action);
 453         }
 454
 455         if (action == Action.DELETE && interfaceUuid != null) {
 456             handleInterfaceEventDelete(intf, dpId);
 457         }
 458     }
 459
 460     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 461         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 462         logger.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 463                 neutronPortUuid, dpId, interfaceUuid.getValue());
 464     }
 465
 466     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 467         // Remove entry from neutronPortToDpIdCache based on interface uuid
 468         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 469             final String currPortUuid = entry.getKey();
 470             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 471                 logger.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 472                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 473                 neutronPortToDpIdCache.remove(currPortUuid);
 474                 break;
 475             }
 476         }
 477     }
 478
 479     //
 480     // Internal helpers
 481     //
 482     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 483
 484         final String networkUUID = neutronPort.getNetworkUUID();
 485         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 486
 487         // If there is no router interface handling the networkUUID, we are done
 488         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 489             return;
 490         }
 491
 492         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 493         // router interface are handled via handleNeutronRouterInterfaceEvent.
 494         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 495             return;
 496         }
 497
 498         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 499         final String providerSegmentationId = neutronNetwork != null ?
 500                                               neutronNetwork.getProviderSegmentationID() : null;
 501         final String tenantMac = neutronPort.getMacAddress();
 502
 503         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 504             tenantMac == null || tenantMac.isEmpty()) {
 505             return;  // done: go no further w/out all the info needed...
 506         }
 507
 508         final Action action = isDelete ? Action.DELETE : Action.ADD;
 509         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 510         if (nodes.isEmpty()) {
 511             logger.trace("updateL3ForNeutronPort has no nodes to work with");
 512         }
 513         for (Node node : nodes) {
 514             final Long dpid = getDpidForIntegrationBridge(node);
 515             if (dpid == null) {
 516                 continue;
 517             }
 518
 519             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 520                 final String tenantIpStr = neutronIP.getIpAddress();
 521                 if (tenantIpStr.isEmpty()) {
 522                     continue;
 523                 }
 524
 525                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 526                 // still needed when routing to subnets non-local to node (bug 2076).
 527                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 528
 529                 // Configure distributed ARP responder
 530                 if (true == flgDistributedARPEnabled) {
 531                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 532                 }
 533             }
 534         }
 535     }
 536
 537     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 538                                            String macAddress, String ipStr,
 539                                            Action actionForNode) {
 540         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 541         // will look at desired action for node.
 542         //
 543         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 544         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 545
 546         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 547             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 548                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 549             return;
 550         }
 551         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 552             logger.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 553                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 554             return;
 555         }
 556
 557         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 558                                                        macAddress, ipStr, actionForNode);
 559         if (status.isSuccess()) {
 560             // Update cache
 561             if (actionForNode == Action.ADD) {
 562                 l3ForwardingCache.add(cacheKey);
 563             } else {
 564                 l3ForwardingCache.remove(cacheKey);
 565             }
 566         }
 567     }
 568
 569     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 570                                              String macAddress,
 571                                              String address,
 572                                              Action actionForNode) {
 573         Status status;
 574         try {
 575             InetAddress inetAddress = InetAddress.getByName(address);
 576             status = l3ForwardingProvider == null ?
 577                      new Status(StatusCode.SUCCESS) :
 578                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 579                                                                       inetAddress, macAddress, actionForNode);
 580         } catch (UnknownHostException e) {
 581             status = new Status(StatusCode.BADREQUEST);
 582         }
 583
 584         if (status.isSuccess()) {
 585             logger.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 586                          l3ForwardingProvider == null ? "skipped" : "programmed",
 587                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 588         } else {
 589             logger.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 590                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 591         }
 592         return status;
 593     }
 594
 595     // --
 596
 597     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 598                                                        Boolean isDelete) {
 599         Preconditions.checkNotNull(destNeutronRouterInterface);
 600
 601         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 602         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 603         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 604         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 605         final NeutronNetwork neutronNetwork = subnet != null ?
 606                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 607         final String destinationSegmentationId = neutronNetwork != null ?
 608                                                  neutronNetwork.getProviderSegmentationID() : null;
 609         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 610         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 611         final String cidr = subnet != null ? subnet.getCidr() : null;
 612         final int mask = getMaskLenFromCidr(cidr);
 613
 614         logger.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 615                      destNeutronRouterInterface, isDelete);
 616
 617         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 618         // the local cache
 619         if (neutronNetwork != null) {
 620             if (macAddress == null || macAddress.isEmpty()) {
 621                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 622             }
 623             if (ipList == null || ipList.isEmpty()) {
 624                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 625             }
 626         }
 627
 628         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 629             cidr == null || cidr.isEmpty() ||
 630             macAddress == null || macAddress.isEmpty() ||
 631             ipList == null || ipList.isEmpty()) {
 632             logger.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 633                          destinationSegmentationId, cidr, macAddress, ipList);
 634             return;  // done: go no further w/out all the info needed...
 635         }
 636
 637         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 638
 639         // Keep cache for finding router's mac from network uuid -- add
 640         //
 641         if (! isDelete) {
 642             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 643             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 644             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 645         }
 646
 647         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 648         if (nodes.isEmpty()) {
 649             logger.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 650         }
 651         for (Node node : nodes) {
 652             final Long dpid = getDpidForIntegrationBridge(node);
 653             if (dpid == null) {
 654                 continue;
 655             }
 656
 657             for (Neutron_IPs neutronIP : ipList) {
 658                 final String ipStr = neutronIP.getIpAddress();
 659                 if (ipStr.isEmpty()) {
 660                     logger.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 661                             node.getNodeId().getValue(), ipStr);
 662                     continue;
 663                 }
 664
 665                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 666                 //
 667                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 668                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 669                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 670                                                               neutronNetwork, destinationSegmentationId,
 671                                                               macAddress, ipStr, mask, actionForNode,
 672                                                               true /*isReflexsive*/);
 673                 }
 674
 675                 if (! isExternal) {
 676                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 677                             actionForNode);
 678                 }
 679                 // Enable ARP responder by default, because router interface needs to be responded always.
 680                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 681             }
 682
 683             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 684             // for the external neutron networks.
 685             //
 686             {
 687                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 688                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 689             }
 690         }
 691
 692         // Keep cache for finding router's mac from network uuid -- remove
 693         //
 694         if (isDelete) {
 695             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 696             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 697             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 698         }
 699     }
 700
 701     private void programFlowForNetworkFromExternal(final Node node,
 702                                                    final Long dpid,
 703                                                    final String destinationSegmentationId,
 704                                                    final String dstMacAddress,
 705                                                    final String destIpStr,
 706                                                    final int destMask,
 707                                                    final Action actionForNode) {
 708         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 709                 dstMacAddress, destIpStr, destMask, actionForNode);
 710     }
 711
 712     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 713                                                            final Long dpid,
 714                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 715                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 716                                                            final NeutronNetwork dstNeutronNetwork,
 717                                                            final String destinationSegmentationId,
 718                                                            final String dstMacAddress,
 719                                                            final String destIpStr,
 720                                                            final int destMask,
 721                                                            final Action actionForNode,
 722                                                            Boolean isReflexsive) {
 723         Preconditions.checkNotNull(srcNeutronRouterInterface);
 724         Preconditions.checkNotNull(dstNeutronRouterInterface);
 725
 726         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 727         if (sourceSubnetId == null) {
 728             logger.error("Could not get provider Subnet ID from router interface {}",
 729                          srcNeutronRouterInterface.getID());
 730             return;
 731         }
 732
 733         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 734         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 735         if (sourceNetworkId == null) {
 736             logger.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 737             return;
 738         }
 739
 740         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 741         if (sourceNetwork == null) {
 742             logger.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 743             return;
 744         }
 745
 746         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 747             // Isolate subnets from different tenants within the same router
 748             return;
 749         }
 750         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 751         if (sourceSegmentationId == null) {
 752             logger.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 753             return;
 754         }
 755         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 756             // Skip 'self'
 757             return;
 758         }
 759
 760         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 761                                      dstMacAddress, destIpStr, destMask, actionForNode);
 762
 763         // Flip roles src->dst; dst->src
 764         if (isReflexsive) {
 765             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 766             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 767             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 768             final String cidr2 = sourceSubnet.getCidr();
 769             final int mask2 = getMaskLenFromCidr(cidr2);
 770
 771             if (cidr2 == null || cidr2.isEmpty() ||
 772                 macAddress2 == null || macAddress2.isEmpty() ||
 773                 ipList2 == null || ipList2.isEmpty()) {
 774                 logger.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 775                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 776                 return;  // done: go no further w/out all the info needed...
 777             }
 778
 779             for (Neutron_IPs neutronIP2 : ipList2) {
 780                 final String ipStr2 = neutronIP2.getIpAddress();
 781                 if (ipStr2.isEmpty()) {
 782                     continue;
 783                 }
 784                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 785                                                           srcNeutronRouterInterface,
 786                                                           sourceNetwork, sourceSegmentationId,
 787                                                           macAddress2, ipStr2, mask2, actionForNode,
 788                                                           false /*isReflexsive*/);
 789             }
 790         }
 791     }
 792
 793     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 794                                               String destinationSegmentationId,
 795                                               String macAddress, String ipStr, int mask,
 796                                               Action actionForNode) {
 797         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 798         // will look at desired action for node.
 799         //
 800         final String cacheKey = node.getNodeId().getValue() + ":" +
 801                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 802                                 ipStr + "/" + Integer.toString(mask);
 803         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 804
 805         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 806             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 807                          " action {} is already done",
 808                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 809                          macAddress, ipStr, mask, actionForNode);
 810             return;
 811         }
 812         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 813             logger.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 814                          " action {} is already done",
 815                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 816                          macAddress, ipStr, mask, actionForNode);
 817             return;
 818         }
 819
 820         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 821                                                           macAddress, ipStr, mask, actionForNode);
 822         if (status.isSuccess()) {
 823             // Update cache
 824             if (actionForNode == Action.ADD) {
 825                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 826                 routerInterfacesCache.add(cacheKey);
 827             } else {
 828                 // TODO: multiTenantAwareRouter.removeInterface(...);
 829                 routerInterfacesCache.remove(cacheKey);
 830             }
 831         }
 832     }
 833
 834     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 835                                                 String destinationSegmentationId,
 836                                                 String macAddress,
 837                                                 String address, int mask,
 838                                                 Action actionForNode) {
 839         Status status;
 840         try {
 841             InetAddress inetAddress = InetAddress.getByName(address);
 842             status = routingProvider == null ?
 843                      new Status(StatusCode.SUCCESS) :
 844                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 845                                                             macAddress, inetAddress, mask, actionForNode);
 846         } catch (UnknownHostException e) {
 847             status = new Status(StatusCode.BADREQUEST);
 848         }
 849
 850         if (status.isSuccess()) {
 851             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 852                          routingProvider == null ? "skipped" : "programmed",
 853                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 854                          actionForNode);
 855         } else {
 856             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 857                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 858                          actionForNode, status);
 859         }
 860         return status;
 861     }
 862
 863     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 864                                            String macAddress, String ipStr,
 865                                            Action action) {
 866         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 867         // will look at desired action for node.
 868         //
 869         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 870         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 871
 872         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 873             logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 874                     dpid, segOrOfPort, macAddress, ipStr, action);
 875             return true;
 876         }
 877         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 878             logger.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 879                     dpid, segOrOfPort, macAddress, ipStr, action);
 880             return true;
 881         }
 882
 883         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 884         if (status.isSuccess()) {
 885             // Update cache
 886             if (action == Action.ADD) {
 887                 staticArpEntryCache.add(cacheKey);
 888             } else {
 889                 staticArpEntryCache.remove(cacheKey);
 890             }
 891             return true;
 892         }
 893         return false;
 894     }
 895
 896     private Status programStaticArpStage2(Long dpid,
 897                                           String segOrOfPort,
 898                                           String macAddress,
 899                                           String address,
 900                                           Action action) {
 901         Status status;
 902         try {
 903             InetAddress inetAddress = InetAddress.getByName(address);
 904             status = arpProvider == null ?
 905                      new Status(StatusCode.SUCCESS) :
 906                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
 907                                                        macAddress, inetAddress, action);
 908         } catch (UnknownHostException e) {
 909             status = new Status(StatusCode.BADREQUEST);
 910         }
 911
 912         if (status.isSuccess()) {
 913             logger.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
 914                          arpProvider == null ? "skipped" : "programmed",
 915                          macAddress, address, dpid, segOrOfPort, action);
 916         } else {
 917             logger.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
 918                          macAddress, address, dpid, segOrOfPort, action, status);
 919         }
 920         return status;
 921     }
 922
 923     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
 924                                                   String matchAddress, String rewriteAddress,
 925                                                   Action action) {
 926         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 927         // will look at desired action for node.
 928         //
 929         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
 930         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
 931
 932         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 933             logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 934                     " action {} is already done",
 935                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 936             return true;
 937         }
 938         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 939             logger.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 940                     " action is already done",
 941                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 942             return true;
 943         }
 944
 945         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
 946                 rewriteAddress, action);
 947         if (status.isSuccess()) {
 948             // Update cache
 949             if (action == Action.ADD) {
 950                 inboundIpRewriteCache.add(cacheKey);
 951             } else {
 952                 inboundIpRewriteCache.remove(cacheKey);
 953             }
 954             return true;
 955         }
 956         return false;
 957     }
 958
 959     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
 960                                                  String matchAddress, String rewriteAddress,
 961                                                  Action action) {
 962         Status status;
 963         try {
 964             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 965             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 966             status = inboundNatProvider == null ?
 967                     new Status(StatusCode.SUCCESS) :
 968                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
 969                             inetMatchAddress, inetRewriteAddress,
 970                             action);
 971         } catch (UnknownHostException e) {
 972             status = new Status(StatusCode.BADREQUEST);
 973         }
 974
 975         if (status.isSuccess()) {
 976             final boolean isSkipped = inboundNatProvider == null;
 977             logger.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
 978                     (isSkipped ? "skipped" : "programmed"),
 979                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 980         } else {
 981             logger.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
 982                          " status:{}",
 983                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
 984                     status);
 985         }
 986         return status;
 987     }
 988
 989     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
 990                                                  Action actionForRewriteExclusion) {
 991         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 992         // will look at desired action for node.
 993         //
 994         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
 995         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
 996
 997         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
 998             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
 999                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1000             return;
1001         }
1002         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1003             logger.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1004                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1005             return;
1006         }
1007
1008         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1009                                                              actionForRewriteExclusion);
1010         if (status.isSuccess()) {
1011             // Update cache
1012             if (actionForRewriteExclusion == Action.ADD) {
1013                     outboundIpRewriteExclusionCache.add(cacheKey);
1014             } else {
1015                     outboundIpRewriteExclusionCache.remove(cacheKey);
1016             }
1017         }
1018     }
1019
1020     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1021                                                    Action actionForNode) {
1022         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1023                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1024
1025         if (status.isSuccess()) {
1026             final boolean isSkipped = outboundNatProvider == null;
1027             logger.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1028                          (isSkipped ? "skipped" : "programmed"),
1029                          cidr, node.getNodeId().getValue(), actionForNode);
1030         } else {
1031             logger.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1032                          cidr, node.getNodeId().getValue(), actionForNode, status);
1033         }
1034         return status;
1035     }
1036
1037     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1038         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1039         // will look at desired action for node.
1040         //
1041         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1042         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1043
1044         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1045             logger.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1046                          "is already done",
1047                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1048             return;
1049         }
1050         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1051             logger.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1052                          "is already done",
1053                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1054             return;
1055         }
1056
1057         Status status = this.programOutboundIpRewriteStage2(fid, action);
1058         if (status.isSuccess()) {
1059             // Update cache
1060             if (action == Action.ADD) {
1061                 outboundIpRewriteCache.add(cacheKey);
1062             } else {
1063                 outboundIpRewriteCache.remove(cacheKey);
1064             }
1065         }
1066     }
1067
1068     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1069         Status status;
1070         try {
1071             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1072             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1073             status = outboundNatProvider == null ?
1074                     new Status(StatusCode.SUCCESS) :
1075                     outboundNatProvider.programIpRewriteRule(
1076                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1077                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1078         } catch (UnknownHostException e) {
1079             status = new Status(StatusCode.BADREQUEST);
1080         }
1081
1082         if (status.isSuccess()) {
1083             final boolean isSkipped = outboundNatProvider == null;
1084             logger.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1085                          " action {}",
1086                          (isSkipped ? "skipped" : "programmed"),
1087                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1088         } else {
1089             logger.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1090                          " action {} status:{}",
1091                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1092         }
1093         return status;
1094     }
1095
1096     private int getMaskLenFromCidr(String cidr) {
1097         if (cidr == null) return 0;
1098         String[] splits = cidr.split("/");
1099         if (splits.length != 2) return 0;
1100
1101         int result;
1102         try {
1103             result = Integer.parseInt(splits[1].trim());
1104         }
1105         catch (NumberFormatException nfe)
1106         {
1107             result = 0;
1108         }
1109         return result;
1110     }
1111
1112     private Long getDpidForIntegrationBridge(Node node) {
1113         // Check if node is integration bridge; and only then return its dpid
1114         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1115             return southbound.getDataPathId(node);
1116         }
1117         return null;
1118     }
1119
1120     /**
1121      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1122      *
1123      * @param ofPort the OF port number
1124      * @return the string with encoded OF port (example format "OFPort|999")
1125      */
1126     public static String encodeExcplicitOFPort(Long ofPort) {
1127         return "OFPort|" + ofPort.toString();
1128     }
1129
1130     @Override
1131     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1132         tenantNetworkManager =
1133                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1134         configurationService =
1135                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1136         arpProvider =
1137                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1138         inboundNatProvider =
1139                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1140         outboundNatProvider =
1141                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1142         routingProvider =
1143                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1144         l3ForwardingProvider =
1145                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1146         nodeCacheManager =
1147                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1148         southbound =
1149                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1150
1151         initL3AdapterMembers();
1152     }
1153
1154     @Override
1155     public void setDependencies(Object impl) {
1156         if (impl instanceof INeutronNetworkCRUD) {
1157             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1158         } else if (impl instanceof INeutronPortCRUD) {
1159             neutronPortCache = (INeutronPortCRUD)impl;
1160         } else if (impl instanceof INeutronSubnetCRUD) {
1161             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1162         } else if (impl instanceof ArpProvider) {
1163             arpProvider = (ArpProvider)impl;
1164         } else if (impl instanceof InboundNatProvider) {
1165             inboundNatProvider = (InboundNatProvider)impl;
1166         } else if (impl instanceof OutboundNatProvider) {
1167             outboundNatProvider = (OutboundNatProvider)impl;
1168         } else if (impl instanceof RoutingProvider) {
1169             routingProvider = (RoutingProvider)impl;
1170         } else if (impl instanceof L3ForwardingProvider) {
1171             l3ForwardingProvider = (L3ForwardingProvider)impl;
1172         }
1173     }
1174 }