openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  11
  12 import org.apache.commons.lang3.tuple.ImmutablePair;
  13 import org.apache.commons.lang3.tuple.Pair;
  14 import org.opendaylight.neutron.spi.INeutronNetworkCRUD;
  15 import org.opendaylight.neutron.spi.INeutronPortCRUD;
  16 import org.opendaylight.neutron.spi.INeutronSubnetCRUD;
  17 import org.opendaylight.neutron.spi.NeutronFloatingIP;
  18 import org.opendaylight.neutron.spi.NeutronNetwork;
  19 import org.opendaylight.neutron.spi.NeutronPort;
  20 import org.opendaylight.neutron.spi.NeutronRouter;
  21 import org.opendaylight.neutron.spi.NeutronRouter_Interface;
  22 import org.opendaylight.neutron.spi.NeutronSubnet;
  23 import org.opendaylight.neutron.spi.Neutron_IPs;
  24 import org.opendaylight.ovsdb.openstack.netvirt.ConfigInterface;
  25 import org.opendaylight.ovsdb.openstack.netvirt.api.*;
  26 import org.opendaylight.ovsdb.utils.servicehelper.ServiceHelper;
  27 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  28 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.rev150105.OvsdbTerminationPointAugmentation;
  29 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  30
  31 import com.google.common.base.Preconditions;
  32 import org.osgi.framework.BundleContext;
  33 import org.osgi.framework.ServiceReference;
  34 import org.slf4j.Logger;
  35 import org.slf4j.LoggerFactory;
  36
  37 import java.net.InetAddress;
  38 import java.net.UnknownHostException;
  39 import java.util.HashMap;
  40 import java.util.HashSet;
  41 import java.util.ArrayList;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Set;
  45
  46 /**
  47  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  48  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  49  * as well as the multi-tenant router forwarding provider.
  50  */
  51 public class NeutronL3Adapter implements ConfigInterface {
  52     private static final Logger LOGGER = LoggerFactory.getLogger(NeutronL3Adapter.class);
  53
  54     // The implementation for each of these services is resolved by the OSGi Service Manager
  55     private volatile ConfigurationService configurationService;
  56     private volatile TenantNetworkManager tenantNetworkManager;
  57     private volatile NodeCacheManager nodeCacheManager;
  58     private volatile INeutronNetworkCRUD neutronNetworkCache;
  59     private volatile INeutronSubnetCRUD neutronSubnetCache;
  60     private volatile INeutronPortCRUD neutronPortCache;
  61     private volatile L3ForwardingProvider l3ForwardingProvider;
  62     private volatile InboundNatProvider inboundNatProvider;
  63     private volatile OutboundNatProvider outboundNatProvider;
  64     private volatile ArpProvider arpProvider;
  65     private volatile RoutingProvider routingProvider;
  66
  67     private class FloatIpData {
  68         private final Long dpid;          // br-int of node where floating ip is associated with tenant port
  69         private final Long ofPort;        // patch port in br-int used to reach br-ex
  70         private final String segId;       // segmentation id of the net where fixed ip is instantiated
  71         private final String macAddress;  // mac address assigned to neutron port of floating ip
  72         private final String floatingIpAddress;
  73         private final String fixedIpAddress;  // ip address given to tenant vm
  74         private final String neutronRouterMac;
  75
  76         FloatIpData(final Long dpid, final Long ofPort, final String segId, final String macAddress,
  77                     final String floatingIpAddress, final String fixedIpAddress, final String neutronRouterMac) {
  78             this.dpid = dpid;
  79             this.ofPort = ofPort;
  80             this.segId = segId;
  81             this.macAddress = macAddress;
  82             this.floatingIpAddress = floatingIpAddress;
  83             this.fixedIpAddress = fixedIpAddress;
  84             this.neutronRouterMac = neutronRouterMac;
  85         }
  86     }
  87
  88     private Set<String> inboundIpRewriteCache;
  89     private Set<String> outboundIpRewriteCache;
  90     private Set<String> outboundIpRewriteExclusionCache;
  91     private Set<String> routerInterfacesCache;
  92     private Set<String> staticArpEntryCache;
  93     private Set<String> l3ForwardingCache;
  94     private Map<String, String> networkIdToRouterMacCache;
  95     private Map<String, List<Neutron_IPs>> networkIdToRouterIpListCache;
  96     private Map<String, NeutronRouter_Interface> subnetIdToRouterInterfaceCache;
  97     private Map<String, Pair<Long, Uuid>> neutronPortToDpIdCache;
  98     private Map<String, FloatIpData> floatIpDataMapCache;
  99     private String externalRouterMac;
 100     private Boolean enabled = false;
 101     private Boolean flgDistributedARPEnabled = true;
 102     private Southbound southbound;
 103
 104     private static final String OWNER_ROUTER_INTERFACE = "network:router_interface";
 105     private static final String OWNER_ROUTER_INTERFACE_DISTRIBUTED = "network:router_interface_distributed";
 106     private static final String OWNER_ROUTER_GATEWAY = "network:router_gateway";
 107     private static final String OWNER_FLOATING_IP = "network:floatingip";
 108     private static final String DEFAULT_EXT_RTR_MAC = "00:00:5E:00:01:01";
 109
 110     public NeutronL3Adapter() {
 111         LOGGER.info(">>>>>> NeutronL3Adapter constructor {}", this.getClass());
 112     }
 113
 114     private void initL3AdapterMembers() {
 115         Preconditions.checkNotNull(configurationService);
 116
 117         if (configurationService.isL3ForwardingEnabled()) {
 118             this.inboundIpRewriteCache = new HashSet<>();
 119             this.outboundIpRewriteCache = new HashSet<>();
 120             this.outboundIpRewriteExclusionCache = new HashSet<>();
 121             this.routerInterfacesCache = new HashSet<>();
 122             this.staticArpEntryCache = new HashSet<>();
 123             this.l3ForwardingCache = new HashSet<>();
 124             this.networkIdToRouterMacCache = new HashMap<>();
 125             this.networkIdToRouterIpListCache = new HashMap<>();
 126             this.subnetIdToRouterInterfaceCache = new HashMap<>();
 127             this.neutronPortToDpIdCache = new HashMap<>();
 128             this.floatIpDataMapCache = new HashMap<>();
 129
 130             this.externalRouterMac = configurationService.getDefaultGatewayMacAddress(null);
 131             if (this.externalRouterMac == null) {
 132                 this.externalRouterMac = DEFAULT_EXT_RTR_MAC;
 133             }
 134
 135             this.enabled = true;
 136             LOGGER.info("OVSDB L3 forwarding is enabled");
 137             if (configurationService.isDistributedArpDisabled()) {
 138                 this.flgDistributedARPEnabled = false;
 139                 LOGGER.info("Distributed ARP responder is disabled");
 140             } else {
 141                 LOGGER.debug("Distributed ARP responder is enabled");
 142             }
 143         } else {
 144             LOGGER.debug("OVSDB L3 forwarding is disabled");
 145         }
 146     }
 147
 148     //
 149     // Callbacks from OVSDB's northbound handlers
 150     //
 151
 152     public void updateExternalRouterMac(final String externalRouterMacUpdate) {
 153         Preconditions.checkNotNull(externalRouterMacUpdate);
 154
 155         flushExistingIpRewrite();
 156         this.externalRouterMac = externalRouterMacUpdate;
 157         rebuildExistingIpRewrite();
 158     }
 159
 160     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 161         LOGGER.debug("Neutron subnet {} event : {}", action, subnet.toString());
 162         if (!this.enabled) {
 163             return;
 164         }
 165     }
 166
 167     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 168         LOGGER.debug("Neutron port {} event : {}", action, neutronPort.toString());
 169         if (!this.enabled) {
 170             return;
 171         }
 172
 173         final boolean isDelete = action == Action.DELETE;
 174
 175         // Treat the port event as a router interface event if the port belongs to router. This is a
 176         // helper for handling cases when handleNeutronRouterInterfaceEvent is not available
 177         //
 178         if (neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE) ||
 179             neutronPort.getDeviceOwner().equalsIgnoreCase(OWNER_ROUTER_INTERFACE_DISTRIBUTED)) {
 180             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 181                 NeutronRouter_Interface neutronRouterInterface =
 182                         new NeutronRouter_Interface(neutronIP.getSubnetUUID(), neutronPort.getPortUUID());
 183                 neutronRouterInterface.setID(neutronIP.getSubnetUUID());  // id of router interface to be same as subnet
 184                 neutronRouterInterface.setTenantID(neutronPort.getTenantID());
 185
 186                 this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 187             }
 188         } else {
 189             // We made it here, port is not used as a router interface. If this is not a delete action, make sure that
 190             // all nodes that are supposed to have a router interface for the port's subnet(s), have it configured. We
 191             // need to do this check here because a router interface is not added to a node until tenant becomes needed
 192             // there.
 193             //
 194             if (!isDelete) {
 195                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 196                     NeutronRouter_Interface neutronRouterInterface =
 197                             subnetIdToRouterInterfaceCache.get(neutronIP.getSubnetUUID());
 198                     if (neutronRouterInterface != null) {
 199                         this.handleNeutronRouterInterfaceEvent(null /*neutronRouter*/, neutronRouterInterface, action);
 200                     }
 201                 }
 202             }
 203             this.updateL3ForNeutronPort(neutronPort, isDelete);
 204         }
 205     }
 206
 207     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 208         LOGGER.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 209         if (!this.enabled) {
 210             return;
 211         }
 212     }
 213
 214     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 215                                                   final NeutronRouter_Interface neutronRouterInterface,
 216                                                   Action action) {
 217         LOGGER.debug("Router interface {} got event {}. Subnet {}",
 218                      neutronRouterInterface.getPortUUID(),
 219                      action,
 220                      neutronRouterInterface.getSubnetUUID());
 221         if (!this.enabled) {
 222             return;
 223         }
 224
 225         final boolean isDelete = action == Action.DELETE;
 226
 227         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, isDelete);
 228
 229         // As neutron router interface is added/removed, we need to iterate through all the neutron ports and
 230         // see if they are affected by l3
 231         //
 232         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 233             boolean currPortShouldBeDeleted = false;
 234             // Note: delete in this case only applies to 1)router interface delete and 2)ports on the same subnet
 235             if (isDelete) {
 236                 for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 237                     if (neutronRouterInterface.getSubnetUUID().equalsIgnoreCase(neutronIP.getSubnetUUID())) {
 238                         currPortShouldBeDeleted = true;
 239                         break;
 240                     }
 241                 }
 242             }
 243             this.updateL3ForNeutronPort(neutronPort, currPortShouldBeDeleted);
 244         }
 245     }
 246
 247     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 248                                              Action actionIn) {
 249         Preconditions.checkNotNull(neutronFloatingIP);
 250
 251         LOGGER.debug(" Floating IP {} {}<->{}, network uuid {}", actionIn,
 252                 neutronFloatingIP.getFixedIPAddress(),
 253                 neutronFloatingIP.getFloatingIPAddress(),
 254                 neutronFloatingIP.getFloatingNetworkUUID());
 255         if (!this.enabled) {
 256             return;
 257         }
 258
 259         Action action;
 260
 261         // Consider action to be delete if getFixedIPAddress is null
 262         //
 263         if (neutronFloatingIP.getFixedIPAddress() == null) {
 264             action = Action.DELETE;
 265         } else {
 266             action = actionIn;
 267         }
 268
 269         // this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 270
 271         if (action != Action.DELETE) {
 272             programFlowsForFloatingIPArpAdd(neutronFloatingIP);  // must be first, as it updates floatIpDataMapCache
 273
 274             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.ADD);
 275             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.ADD);
 276         } else {
 277             programFlowsForFloatingIPOutbound(neutronFloatingIP, Action.DELETE);
 278             programFlowsForFloatingIPInbound(neutronFloatingIP, Action.DELETE);
 279
 280             programFlowsForFloatingIPArpDelete(neutronFloatingIP.getID()); // must be last, as it updates floatIpDataMapCache
 281         }
 282     }
 283
 284     private void programFlowsForFloatingIPInbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 285         Preconditions.checkNotNull(neutronFloatingIP);
 286
 287         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 288         if (fid == null) {
 289             LOGGER.trace("programFlowsForFloatingIPInboundAdd {} for {} uuid {} not in local cache",
 290                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 291             return;
 292         }
 293         programInboundIpRewriteStage1(fid.dpid, fid.ofPort, fid.segId, fid.floatingIpAddress, fid.fixedIpAddress,
 294                                       action);
 295     }
 296
 297     private void programFlowsForFloatingIPOutbound(final NeutronFloatingIP neutronFloatingIP, final Action action) {
 298         Preconditions.checkNotNull(neutronFloatingIP);
 299
 300         final FloatIpData fid = floatIpDataMapCache.get(neutronFloatingIP.getID());
 301         if (fid == null) {
 302             LOGGER.trace("programFlowsForFloatingIPOutbound {} for {} uuid {} not in local cache",
 303                     action, neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 304             return;
 305         }
 306         programOutboundIpRewriteStage1(fid, action);
 307     }
 308
 309     private void flushExistingIpRewrite() {
 310         for (FloatIpData fid : floatIpDataMapCache.values()) {
 311             programOutboundIpRewriteStage1(fid, Action.DELETE);
 312         }
 313     }
 314
 315     private void rebuildExistingIpRewrite() {
 316         for (FloatIpData fid : floatIpDataMapCache.values()) {
 317             programOutboundIpRewriteStage1(fid, Action.ADD);
 318         }
 319     }
 320
 321     private void programFlowsForFloatingIPArpAdd(final NeutronFloatingIP neutronFloatingIP) {
 322         Preconditions.checkNotNull(neutronFloatingIP);
 323         Preconditions.checkNotNull(neutronFloatingIP.getFixedIPAddress());
 324         Preconditions.checkNotNull(neutronFloatingIP.getFloatingIPAddress());
 325
 326         if (floatIpDataMapCache.get(neutronFloatingIP.getID()) != null) {
 327             LOGGER.trace("programFlowsForFloatingIPArpAdd for neutronFloatingIP {} uuid {} is already done",
 328                     neutronFloatingIP.getFloatingIPAddress(), neutronFloatingIP.getID());
 329             return;
 330         }
 331
 332         // find bridge Node where floating ip is configured by looking up cache for its port
 333         final NeutronPort neutronPortForFloatIp = findNeutronPortForFloatingIp(neutronFloatingIP.getID());
 334         final String neutronTenantPortUuid = neutronFloatingIP.getPortUUID();
 335         final Pair<Long, Uuid> nodeIfPair = neutronPortToDpIdCache.get(neutronTenantPortUuid);
 336         final String floatingIpMac = neutronPortForFloatIp == null ? null : neutronPortForFloatIp.getMacAddress();
 337         final String fixedIpAddress = neutronFloatingIP.getFixedIPAddress();
 338         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 339
 340         final NeutronPort tenantNeutronPort = neutronPortCache.getPort(neutronTenantPortUuid);
 341         final NeutronNetwork tenantNeutronNetwork = tenantNeutronPort != null ?
 342                 neutronNetworkCache.getNetwork(tenantNeutronPort.getNetworkUUID()) : null;
 343         final String providerSegmentationId = tenantNeutronNetwork != null ?
 344                 tenantNeutronNetwork.getProviderSegmentationID() : null;
 345         final String neutronRouterMac = tenantNeutronNetwork != null ?
 346                 networkIdToRouterMacCache.get(tenantNeutronNetwork.getID()) : null;
 347
 348         if (nodeIfPair == null || neutronTenantPortUuid == null ||
 349                 providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 350                 floatingIpMac == null || floatingIpMac.isEmpty() ||
 351                 neutronRouterMac == null || neutronRouterMac.isEmpty()) {
 352             LOGGER.trace("Floating IP {}<->{}, incomplete floatPort {} tenantPortUuid {} seg {} mac {} rtrMac {}",
 353                     fixedIpAddress,
 354                     floatingIpAddress,
 355                     neutronPortForFloatIp,
 356                     neutronTenantPortUuid,
 357                     providerSegmentationId,
 358                     floatingIpMac,
 359                     neutronRouterMac);
 360             return;
 361         }
 362
 363         // get ofport for patch port in br-int
 364         final Long dpId = nodeIfPair.getLeft();
 365         final Long ofPort = findOFPortForExtPatch(dpId);
 366         if (ofPort == null) {
 367             LOGGER.warn("Unable to locate OF port of patch port to connect floating ip to external bridge. dpid {}",
 368                     dpId);
 369             return;
 370         }
 371
 372         // Respond to ARPs for the floating ip address by default, via the patch port that connects br-int to br-ex
 373         //
 374         if (programStaticArpStage1(dpId, encodeExcplicitOFPort(ofPort), floatingIpMac, floatingIpAddress,
 375                 Action.ADD)) {
 376             final FloatIpData floatIpData = new FloatIpData(dpId, ofPort, providerSegmentationId, floatingIpMac,
 377                     floatingIpAddress, fixedIpAddress, neutronRouterMac);
 378             floatIpDataMapCache.put(neutronFloatingIP.getID(), floatIpData);
 379             LOGGER.info("Floating IP {}<->{} programmed ARP mac {} on OFport {} seg {} dpid {}",
 380                     neutronFloatingIP.getFixedIPAddress(), neutronFloatingIP.getFloatingIPAddress(),
 381                     floatingIpMac, ofPort, providerSegmentationId, dpId);
 382         }
 383     }
 384
 385     private void programFlowsForFloatingIPArpDelete(final String neutronFloatingIPUuid) {
 386         final FloatIpData floatIpData = floatIpDataMapCache.get(neutronFloatingIPUuid);
 387         if (floatIpData == null) {
 388             LOGGER.trace("programFlowsForFloatingIPArpDelete for uuid {} is not needed", neutronFloatingIPUuid);
 389             return;
 390         }
 391
 392         if (programStaticArpStage1(floatIpData.dpid, encodeExcplicitOFPort(floatIpData.ofPort), floatIpData.macAddress,
 393                 floatIpData.floatingIpAddress, Action.DELETE)) {
 394             floatIpDataMapCache.remove(neutronFloatingIPUuid);
 395             LOGGER.info("Floating IP {} un-programmed ARP mac {} on {} dpid {}",
 396                     floatIpData.floatingIpAddress, floatIpData.macAddress, floatIpData.ofPort, floatIpData.dpid);
 397         }
 398     }
 399
 400     private final NeutronPort findNeutronPortForFloatingIp(final String floatingIpUuid) {
 401         for (NeutronPort neutronPort : neutronPortCache.getAllPorts()) {
 402             if (neutronPort.getDeviceOwner().equals(OWNER_FLOATING_IP) &&
 403                     neutronPort.getDeviceID().equals(floatingIpUuid)) {
 404                 return neutronPort;
 405             }
 406         }
 407         return null;
 408     }
 409
 410     private final Long findOFPortForExtPatch(Long dpId) {
 411         final String brInt = configurationService.getIntegrationBridgeName();
 412         final String brExt = configurationService.getExternalBridgeName();
 413         final String portNameInt = configurationService.getPatchPortName(new ImmutablePair<>(brInt, brExt));
 414
 415         Preconditions.checkNotNull(dpId);
 416         Preconditions.checkNotNull(portNameInt);
 417
 418         final long dpidPrimitive = dpId.longValue();
 419         for (Node node : nodeCacheManager.getBridgeNodes()) {
 420             if (dpidPrimitive == southbound.getDataPathId(node)) {
 421                 final OvsdbTerminationPointAugmentation terminationPointOfBridge =
 422                         southbound.getTerminationPointOfBridge(node, portNameInt);
 423                 return terminationPointOfBridge == null ? null : terminationPointOfBridge.getOfport();
 424             }
 425         }
 426         return null;
 427     }
 428
 429     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 430         LOGGER.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 431         if (!this.enabled) {
 432             return;
 433         }
 434     }
 435
 436     //
 437     // Callbacks from OVSDB's southbound handler
 438     //
 439     public void handleInterfaceEvent(final Node bridgeNode, final OvsdbTerminationPointAugmentation intf,
 440                                      final NeutronNetwork neutronNetwork, Action action) {
 441         LOGGER.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 442                      action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork);
 443         if (!this.enabled) {
 444             return;
 445         }
 446
 447         final NeutronPort neutronPort = tenantNetworkManager.getTenantPort(intf);
 448         final Long dpId = getDpidForIntegrationBridge(bridgeNode);
 449         final Uuid interfaceUuid = intf.getInterfaceUuid();
 450
 451         LOGGER.trace("southbound interface {} node:{} interface:{}, neutronNetwork:{} port:{} dpid:{} intfUuid:{}",
 452                 action, bridgeNode.getNodeId().getValue(), intf.getName(), neutronNetwork, neutronPort, dpId, interfaceUuid);
 453
 454         if (neutronPort != null) {
 455             final String neutronPortUuid = neutronPort.getPortUUID();
 456
 457             if (action != Action.DELETE && neutronPortToDpIdCache.get(neutronPortUuid) == null &&
 458                     dpId != null && interfaceUuid != null) {
 459                 handleInterfaceEventAdd(neutronPortUuid, dpId, interfaceUuid);
 460             }
 461
 462             handleNeutronPortEvent(neutronPort, action);
 463         }
 464
 465         if (action == Action.DELETE && interfaceUuid != null) {
 466             handleInterfaceEventDelete(intf, dpId);
 467         }
 468     }
 469
 470     private void handleInterfaceEventAdd(final String neutronPortUuid, Long dpId, final Uuid interfaceUuid) {
 471         neutronPortToDpIdCache.put(neutronPortUuid, new ImmutablePair<>(dpId, interfaceUuid));
 472         LOGGER.debug("handleInterfaceEvent add cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 473                 neutronPortUuid, dpId, interfaceUuid.getValue());
 474     }
 475
 476     private void handleInterfaceEventDelete(final OvsdbTerminationPointAugmentation intf, final Long dpId) {
 477         // Remove entry from neutronPortToDpIdCache based on interface uuid
 478         for (Map.Entry<String, Pair<Long, Uuid>> entry : neutronPortToDpIdCache.entrySet()) {
 479             final String currPortUuid = entry.getKey();
 480             if (intf.getInterfaceUuid().equals(entry.getValue().getRight())) {
 481                 LOGGER.debug("handleInterfaceEventDelete remove cache entry NeutronPortUuid {} : dpid {}, ifUuid {}",
 482                         currPortUuid, dpId, intf.getInterfaceUuid().getValue());
 483                 neutronPortToDpIdCache.remove(currPortUuid);
 484                 break;
 485             }
 486         }
 487     }
 488
 489     //
 490     // Internal helpers
 491     //
 492     private void updateL3ForNeutronPort(final NeutronPort neutronPort, final boolean isDelete) {
 493
 494         final String networkUUID = neutronPort.getNetworkUUID();
 495         final String routerMacAddress = networkIdToRouterMacCache.get(networkUUID);
 496
 497         // If there is no router interface handling the networkUUID, we are done
 498         if (routerMacAddress == null || routerMacAddress.isEmpty()) {
 499             return;
 500         }
 501
 502         // If this is the neutron port for the router interface itself, ignore it as well. Ports that represent the
 503         // router interface are handled via handleNeutronRouterInterfaceEvent.
 504         if (routerMacAddress.equalsIgnoreCase(neutronPort.getMacAddress())) {
 505             return;
 506         }
 507
 508         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 509         final String providerSegmentationId = neutronNetwork != null ?
 510                                               neutronNetwork.getProviderSegmentationID() : null;
 511         final String tenantMac = neutronPort.getMacAddress();
 512
 513         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 514             tenantMac == null || tenantMac.isEmpty()) {
 515             // done: go no further w/out all the info needed...
 516             return;
 517         }
 518
 519         final Action action = isDelete ? Action.DELETE : Action.ADD;
 520         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 521         if (nodes.isEmpty()) {
 522             LOGGER.trace("updateL3ForNeutronPort has no nodes to work with");
 523         }
 524         for (Node node : nodes) {
 525             final Long dpid = getDpidForIntegrationBridge(node);
 526             if (dpid == null) {
 527                 continue;
 528             }
 529
 530             for (Neutron_IPs neutronIP : neutronPort.getFixedIPs()) {
 531                 final String tenantIpStr = neutronIP.getIpAddress();
 532                 if (tenantIpStr.isEmpty()) {
 533                     continue;
 534                 }
 535
 536                 // Configure L3 fwd. We do that regardless of tenant network present, because these rules are
 537                 // still needed when routing to subnets non-local to node (bug 2076).
 538                 programL3ForwardingStage1(node, dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 539
 540                 // Configure distributed ARP responder
 541                 if (true == flgDistributedARPEnabled) {
 542                     programStaticArpStage1(dpid, providerSegmentationId, tenantMac, tenantIpStr, action);
 543                 }
 544             }
 545         }
 546     }
 547
 548     private void programL3ForwardingStage1(Node node, Long dpid, String providerSegmentationId,
 549                                            String macAddress, String ipStr,
 550                                            Action actionForNode) {
 551         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 552         // will look at desired action for node.
 553         //
 554         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + ipStr;
 555         final Boolean isProgrammed = l3ForwardingCache.contains(cacheKey);
 556
 557         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 558             LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 559                          node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 560             return;
 561         }
 562         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 563             LOGGER.trace("programL3ForwardingStage1 for node {} providerId {} mac {} ip {} action {} is already done",
 564                     node.getNodeId().getValue(), providerSegmentationId, macAddress, ipStr, actionForNode);
 565             return;
 566         }
 567
 568         Status status = this.programL3ForwardingStage2(node, dpid, providerSegmentationId,
 569                                                        macAddress, ipStr, actionForNode);
 570         if (status.isSuccess()) {
 571             // Update cache
 572             if (actionForNode == Action.ADD) {
 573                 l3ForwardingCache.add(cacheKey);
 574             } else {
 575                 l3ForwardingCache.remove(cacheKey);
 576             }
 577         }
 578     }
 579
 580     private Status programL3ForwardingStage2(Node node, Long dpid, String providerSegmentationId,
 581                                              String macAddress,
 582                                              String address,
 583                                              Action actionForNode) {
 584         Status status;
 585         try {
 586             InetAddress inetAddress = InetAddress.getByName(address);
 587             status = l3ForwardingProvider == null ?
 588                      new Status(StatusCode.SUCCESS) :
 589                      l3ForwardingProvider.programForwardingTableEntry(dpid, providerSegmentationId,
 590                                                                       inetAddress, macAddress, actionForNode);
 591         } catch (UnknownHostException e) {
 592             status = new Status(StatusCode.BADREQUEST);
 593         }
 594
 595         if (status.isSuccess()) {
 596             LOGGER.debug("ProgramL3Forwarding {} for mac:{} addr:{} node:{} action:{}",
 597                          l3ForwardingProvider == null ? "skipped" : "programmed",
 598                          macAddress, address, node.getNodeId().getValue(), actionForNode);
 599         } else {
 600             LOGGER.error("ProgramL3Forwarding failed for mac:{} addr:{} node:{} action:{} status:{}",
 601                          macAddress, address, node.getNodeId().getValue(), actionForNode, status);
 602         }
 603         return status;
 604     }
 605
 606     // --
 607
 608     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface destNeutronRouterInterface,
 609                                                        Boolean isDelete) {
 610         Preconditions.checkNotNull(destNeutronRouterInterface);
 611
 612         final NeutronPort neutronPort = neutronPortCache.getPort(destNeutronRouterInterface.getPortUUID());
 613         String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 614         List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 615         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(destNeutronRouterInterface.getSubnetUUID());
 616         final NeutronNetwork neutronNetwork = subnet != null ?
 617                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 618         final String destinationSegmentationId = neutronNetwork != null ?
 619                                                  neutronNetwork.getProviderSegmentationID() : null;
 620         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 621         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 622         final String cidr = subnet != null ? subnet.getCidr() : null;
 623         final int mask = getMaskLenFromCidr(cidr);
 624
 625         LOGGER.trace("programFlowsForNeutronRouterInterface called for interface {} isDelete {}",
 626                      destNeutronRouterInterface, isDelete);
 627
 628         // in delete path, mac address as well as ip address are not provided. Being so, let's find them from
 629         // the local cache
 630         if (neutronNetwork != null) {
 631             if (macAddress == null || macAddress.isEmpty()) {
 632                 macAddress = networkIdToRouterMacCache.get(neutronNetwork.getNetworkUUID());
 633             }
 634             if (ipList == null || ipList.isEmpty()) {
 635                 ipList = networkIdToRouterIpListCache.get(neutronNetwork.getNetworkUUID());
 636             }
 637         }
 638
 639         if (destinationSegmentationId == null || destinationSegmentationId.isEmpty() ||
 640             cidr == null || cidr.isEmpty() ||
 641             macAddress == null || macAddress.isEmpty() ||
 642             ipList == null || ipList.isEmpty()) {
 643             LOGGER.debug("programFlowsForNeutronRouterInterface is bailing seg:{} cidr:{} mac:{}  ip:{}",
 644                          destinationSegmentationId, cidr, macAddress, ipList);
 645             // done: go no further w/out all the info needed...
 646             return;
 647         }
 648
 649         final Action actionForNode = isDelete ? Action.DELETE : Action.ADD;
 650
 651         // Keep cache for finding router's mac from network uuid -- add
 652         //
 653         if (! isDelete) {
 654             networkIdToRouterMacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 655             networkIdToRouterIpListCache.put(neutronNetwork.getNetworkUUID(), new ArrayList<>(ipList));
 656             subnetIdToRouterInterfaceCache.put(subnet.getSubnetUUID(), destNeutronRouterInterface);
 657         }
 658
 659         List<Node> nodes = nodeCacheManager.getBridgeNodes();
 660         if (nodes.isEmpty()) {
 661             LOGGER.trace("programFlowsForNeutronRouterInterface has no nodes to work with");
 662         }
 663         for (Node node : nodes) {
 664             final Long dpid = getDpidForIntegrationBridge(node);
 665             if (dpid == null) {
 666                 continue;
 667             }
 668
 669             for (Neutron_IPs neutronIP : ipList) {
 670                 final String ipStr = neutronIP.getIpAddress();
 671                 if (ipStr.isEmpty()) {
 672                     LOGGER.debug("programFlowsForNeutronRouterInterface is skipping node {} ip {}",
 673                             node.getNodeId().getValue(), ipStr);
 674                     continue;
 675                 }
 676
 677                 // Iterate through all other interfaces and add/remove reflexive flows to this interface
 678                 //
 679                 for (NeutronRouter_Interface srcNeutronRouterInterface : subnetIdToRouterInterfaceCache.values()) {
 680                     programFlowsForNeutronRouterInterfacePair(node, dpid,
 681                                                               srcNeutronRouterInterface, destNeutronRouterInterface,
 682                                                               neutronNetwork, destinationSegmentationId,
 683                                                               macAddress, ipStr, mask, actionForNode,
 684                                                               true /*isReflexsive*/);
 685                 }
 686
 687                 if (! isExternal) {
 688                     programFlowForNetworkFromExternal(node, dpid, destinationSegmentationId, macAddress, ipStr, mask,
 689                             actionForNode);
 690                 }
 691                 // Enable ARP responder by default, because router interface needs to be responded always.
 692                 programStaticArpStage1(dpid, destinationSegmentationId, macAddress, ipStr, actionForNode);
 693             }
 694
 695             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 696             // for the external neutron networks.
 697             //
 698             {
 699                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 700                 programIpRewriteExclusionStage1(node, dpid, destinationSegmentationId, cidr, actionForRewriteExclusion);
 701             }
 702         }
 703
 704         // Keep cache for finding router's mac from network uuid -- remove
 705         //
 706         if (isDelete) {
 707             networkIdToRouterMacCache.remove(neutronNetwork.getNetworkUUID());
 708             networkIdToRouterIpListCache.remove(neutronNetwork.getNetworkUUID());
 709             subnetIdToRouterInterfaceCache.remove(subnet.getSubnetUUID());
 710         }
 711     }
 712
 713     private void programFlowForNetworkFromExternal(final Node node,
 714                                                    final Long dpid,
 715                                                    final String destinationSegmentationId,
 716                                                    final String dstMacAddress,
 717                                                    final String destIpStr,
 718                                                    final int destMask,
 719                                                    final Action actionForNode) {
 720         programRouterInterfaceStage1(node, dpid, Constants.EXTERNAL_NETWORK, destinationSegmentationId,
 721                 dstMacAddress, destIpStr, destMask, actionForNode);
 722     }
 723
 724     private void programFlowsForNeutronRouterInterfacePair(final Node node,
 725                                                            final Long dpid,
 726                                                            final NeutronRouter_Interface srcNeutronRouterInterface,
 727                                                            final NeutronRouter_Interface dstNeutronRouterInterface,
 728                                                            final NeutronNetwork dstNeutronNetwork,
 729                                                            final String destinationSegmentationId,
 730                                                            final String dstMacAddress,
 731                                                            final String destIpStr,
 732                                                            final int destMask,
 733                                                            final Action actionForNode,
 734                                                            Boolean isReflexsive) {
 735         Preconditions.checkNotNull(srcNeutronRouterInterface);
 736         Preconditions.checkNotNull(dstNeutronRouterInterface);
 737
 738         final String sourceSubnetId = srcNeutronRouterInterface.getSubnetUUID();
 739         if (sourceSubnetId == null) {
 740             LOGGER.error("Could not get provider Subnet ID from router interface {}",
 741                          srcNeutronRouterInterface.getID());
 742             return;
 743         }
 744
 745         final NeutronSubnet sourceSubnet = neutronSubnetCache.getSubnet(sourceSubnetId);
 746         final String sourceNetworkId = sourceSubnet == null ? null : sourceSubnet.getNetworkUUID();
 747         if (sourceNetworkId == null) {
 748             LOGGER.error("Could not get provider Network ID from subnet {}", sourceSubnetId);
 749             return;
 750         }
 751
 752         final NeutronNetwork sourceNetwork = neutronNetworkCache.getNetwork(sourceNetworkId);
 753         if (sourceNetwork == null) {
 754             LOGGER.error("Could not get provider Network for Network ID {}", sourceNetworkId);
 755             return;
 756         }
 757
 758         if (! sourceNetwork.getTenantID().equals(dstNeutronNetwork.getTenantID())) {
 759             // Isolate subnets from different tenants within the same router
 760             return;
 761         }
 762         final String sourceSegmentationId = sourceNetwork.getProviderSegmentationID();
 763         if (sourceSegmentationId == null) {
 764             LOGGER.error("Could not get provider Segmentation ID for Subnet {}", sourceSubnetId);
 765             return;
 766         }
 767         if (sourceSegmentationId.equals(destinationSegmentationId)) {
 768             // Skip 'self'
 769             return;
 770         }
 771
 772         programRouterInterfaceStage1(node, dpid, sourceSegmentationId, destinationSegmentationId,
 773                                      dstMacAddress, destIpStr, destMask, actionForNode);
 774
 775         // Flip roles src->dst; dst->src
 776         if (isReflexsive) {
 777             final NeutronPort sourceNeutronPort = neutronPortCache.getPort(srcNeutronRouterInterface.getPortUUID());
 778             final String macAddress2 = sourceNeutronPort != null ? sourceNeutronPort.getMacAddress() : null;
 779             final List<Neutron_IPs> ipList2 = sourceNeutronPort != null ? sourceNeutronPort.getFixedIPs() : null;
 780             final String cidr2 = sourceSubnet.getCidr();
 781             final int mask2 = getMaskLenFromCidr(cidr2);
 782
 783             if (cidr2 == null || cidr2.isEmpty() ||
 784                 macAddress2 == null || macAddress2.isEmpty() ||
 785                 ipList2 == null || ipList2.isEmpty()) {
 786                 LOGGER.trace("programFlowsForNeutronRouterInterfacePair reflexive is bailing seg:{} cidr:{} mac:{} ip:{}",
 787                              sourceSegmentationId, cidr2, macAddress2, ipList2);
 788                 // done: go no further w/out all the info needed...
 789                 return;
 790             }
 791
 792             for (Neutron_IPs neutronIP2 : ipList2) {
 793                 final String ipStr2 = neutronIP2.getIpAddress();
 794                 if (ipStr2.isEmpty()) {
 795                     continue;
 796                 }
 797                 programFlowsForNeutronRouterInterfacePair(node, dpid, dstNeutronRouterInterface,
 798                                                           srcNeutronRouterInterface,
 799                                                           sourceNetwork, sourceSegmentationId,
 800                                                           macAddress2, ipStr2, mask2, actionForNode,
 801                                                           false /*isReflexsive*/);
 802             }
 803         }
 804     }
 805
 806     private void programRouterInterfaceStage1(Node node, Long dpid, String sourceSegmentationId,
 807                                               String destinationSegmentationId,
 808                                               String macAddress, String ipStr, int mask,
 809                                               Action actionForNode) {
 810         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 811         // will look at desired action for node.
 812         //
 813         final String cacheKey = node.getNodeId().getValue() + ":" +
 814                                 sourceSegmentationId + ":" + destinationSegmentationId + ":" +
 815                                 ipStr + "/" + Integer.toString(mask);
 816         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 817
 818         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE) {
 819             LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 820                          " action {} is already done",
 821                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 822                          macAddress, ipStr, mask, actionForNode);
 823             return;
 824         }
 825         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE) {
 826             LOGGER.trace("programRouterInterfaceStage1 for node {} sourceSegId {} destSegId {} mac {} ip {} mask {}" +
 827                          " action {} is already done",
 828                          node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 829                          macAddress, ipStr, mask, actionForNode);
 830             return;
 831         }
 832
 833         Status status = this.programRouterInterfaceStage2(node, dpid, sourceSegmentationId, destinationSegmentationId,
 834                                                           macAddress, ipStr, mask, actionForNode);
 835         if (status.isSuccess()) {
 836             // Update cache
 837             if (actionForNode == Action.ADD) {
 838                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 839                 routerInterfacesCache.add(cacheKey);
 840             } else {
 841                 // TODO: multiTenantAwareRouter.removeInterface(...);
 842                 routerInterfacesCache.remove(cacheKey);
 843             }
 844         }
 845     }
 846
 847     private Status programRouterInterfaceStage2(Node node, Long dpid, String sourceSegmentationId,
 848                                                 String destinationSegmentationId,
 849                                                 String macAddress,
 850                                                 String address, int mask,
 851                                                 Action actionForNode) {
 852         Status status;
 853         try {
 854             InetAddress inetAddress = InetAddress.getByName(address);
 855             status = routingProvider == null ?
 856                      new Status(StatusCode.SUCCESS) :
 857                      routingProvider.programRouterInterface(dpid, sourceSegmentationId, destinationSegmentationId,
 858                                                             macAddress, inetAddress, mask, actionForNode);
 859         } catch (UnknownHostException e) {
 860             status = new Status(StatusCode.BADREQUEST);
 861         }
 862
 863         if (status.isSuccess()) {
 864             LOGGER.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{}",
 865                          routingProvider == null ? "skipped" : "programmed",
 866                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 867                          actionForNode);
 868         } else {
 869             LOGGER.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} srcTunId:{} destTunId:{} action:{} status:{}",
 870                          macAddress, address, mask, node.getNodeId().getValue(), sourceSegmentationId, destinationSegmentationId,
 871                          actionForNode, status);
 872         }
 873         return status;
 874     }
 875
 876     private boolean programStaticArpStage1(Long dpid, String segOrOfPort,
 877                                            String macAddress, String ipStr,
 878                                            Action action) {
 879         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 880         // will look at desired action for node.
 881         //
 882         final String cacheKey = dpid + ":" + segOrOfPort + ":" + ipStr;
 883         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 884
 885         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 886             LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 887                     dpid, segOrOfPort, macAddress, ipStr, action);
 888             return true;
 889         }
 890         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 891             LOGGER.trace("programStaticArpStage1 dpid {} segOrOfPort {} mac {} ip {} action {} is already done",
 892                     dpid, segOrOfPort, macAddress, ipStr, action);
 893             return true;
 894         }
 895
 896         Status status = this.programStaticArpStage2(dpid, segOrOfPort, macAddress, ipStr, action);
 897         if (status.isSuccess()) {
 898             // Update cache
 899             if (action == Action.ADD) {
 900                 staticArpEntryCache.add(cacheKey);
 901             } else {
 902                 staticArpEntryCache.remove(cacheKey);
 903             }
 904             return true;
 905         }
 906         return false;
 907     }
 908
 909     private Status programStaticArpStage2(Long dpid,
 910                                           String segOrOfPort,
 911                                           String macAddress,
 912                                           String address,
 913                                           Action action) {
 914         Status status;
 915         try {
 916             InetAddress inetAddress = InetAddress.getByName(address);
 917             status = arpProvider == null ?
 918                      new Status(StatusCode.SUCCESS) :
 919                      arpProvider.programStaticArpEntry(dpid, segOrOfPort,
 920                                                        macAddress, inetAddress, action);
 921         } catch (UnknownHostException e) {
 922             status = new Status(StatusCode.BADREQUEST);
 923         }
 924
 925         if (status.isSuccess()) {
 926             LOGGER.debug("ProgramStaticArp {} for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{}",
 927                          arpProvider == null ? "skipped" : "programmed",
 928                          macAddress, address, dpid, segOrOfPort, action);
 929         } else {
 930             LOGGER.error("ProgramStaticArp failed for mac:{} addr:{} dpid:{} segOrOfPort:{} action:{} status:{}",
 931                          macAddress, address, dpid, segOrOfPort, action, status);
 932         }
 933         return status;
 934     }
 935
 936     private boolean programInboundIpRewriteStage1(Long dpid, Long inboundOFPort, String providerSegmentationId,
 937                                                   String matchAddress, String rewriteAddress,
 938                                                   Action action) {
 939         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 940         // will look at desired action for node.
 941         //
 942         final String cacheKey = dpid + ":" + inboundOFPort + ":" + providerSegmentationId + ":" + matchAddress;
 943         final Boolean isProgrammed = inboundIpRewriteCache.contains(cacheKey);
 944
 945         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
 946             LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 947                     " action {} is already done",
 948                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 949             return true;
 950         }
 951         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
 952             LOGGER.trace("programInboundIpRewriteStage1 dpid {} OFPort {} seg {} matchAddress {} rewriteAddress {}" +
 953                     " action is already done",
 954                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 955             return true;
 956         }
 957
 958         Status status = programInboundIpRewriteStage2(dpid, inboundOFPort, providerSegmentationId, matchAddress,
 959                 rewriteAddress, action);
 960         if (status.isSuccess()) {
 961             // Update cache
 962             if (action == Action.ADD) {
 963                 inboundIpRewriteCache.add(cacheKey);
 964             } else {
 965                 inboundIpRewriteCache.remove(cacheKey);
 966             }
 967             return true;
 968         }
 969         return false;
 970     }
 971
 972     private Status programInboundIpRewriteStage2(Long dpid, Long inboundOFPort, String providerSegmentationId,
 973                                                  String matchAddress, String rewriteAddress,
 974                                                  Action action) {
 975         Status status;
 976         try {
 977             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 978             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 979             status = inboundNatProvider == null ?
 980                     new Status(StatusCode.SUCCESS) :
 981                     inboundNatProvider.programIpRewriteRule(dpid, inboundOFPort, providerSegmentationId,
 982                             inetMatchAddress, inetRewriteAddress,
 983                             action);
 984         } catch (UnknownHostException e) {
 985             status = new Status(StatusCode.BADREQUEST);
 986         }
 987
 988         if (status.isSuccess()) {
 989             final boolean isSkipped = inboundNatProvider == null;
 990             LOGGER.debug("programInboundIpRewriteStage2 {} for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}",
 991                     isSkipped ? "skipped" : "programmed",
 992                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action);
 993         } else {
 994             LOGGER.error("programInboundIpRewriteStage2 failed for dpid:{} ofPort:{} seg:{} match:{} rewrite:{} action:{}" +
 995                          " status:{}",
 996                     dpid, inboundOFPort, providerSegmentationId, matchAddress, rewriteAddress, action,
 997                     status);
 998         }
 999         return status;
1000     }
1001
1002     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId, String cidr,
1003                                                  Action actionForRewriteExclusion) {
1004         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1005         // will look at desired action for node.
1006         //
1007         final String cacheKey = node.getNodeId().getValue() + ":" + providerSegmentationId + ":" + cidr;
1008         final Boolean isProgrammed = outboundIpRewriteExclusionCache.contains(cacheKey);
1009
1010         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE) {
1011             LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1012                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1013             return;
1014         }
1015         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE) {
1016             LOGGER.trace("programIpRewriteExclusionStage1 node {} providerId {} cidr {} action {} is already done",
1017                          node.getNodeId().getValue(), providerSegmentationId, cidr, actionForRewriteExclusion);
1018             return;
1019         }
1020
1021         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
1022                                                              actionForRewriteExclusion);
1023         if (status.isSuccess()) {
1024             // Update cache
1025             if (actionForRewriteExclusion == Action.ADD) {
1026                     outboundIpRewriteExclusionCache.add(cacheKey);
1027             } else {
1028                     outboundIpRewriteExclusionCache.remove(cacheKey);
1029             }
1030         }
1031     }
1032
1033     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
1034                                                    Action actionForNode) {
1035         final Status status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
1036                 outboundNatProvider.programIpRewriteExclusion(dpid, providerSegmentationId, cidr, actionForNode);
1037
1038         if (status.isSuccess()) {
1039             final boolean isSkipped = outboundNatProvider == null;
1040             LOGGER.debug("IpRewriteExclusion {} for cidr:{} node:{} action:{}",
1041                          isSkipped ? "skipped" : "programmed",
1042                          cidr, node.getNodeId().getValue(), actionForNode);
1043         } else {
1044             LOGGER.error("IpRewriteExclusion failed for cidr:{} node:{} action:{} status:{}",
1045                          cidr, node.getNodeId().getValue(), actionForNode, status);
1046         }
1047         return status;
1048     }
1049
1050     private void programOutboundIpRewriteStage1(FloatIpData fid, Action action) {
1051         // Based on the local cache, figure out whether programming needs to occur. To do this, we
1052         // will look at desired action for node.
1053         //
1054         final String cacheKey = fid.dpid + ":" + fid.segId + ":" + fid.fixedIpAddress;
1055         final Boolean isProgrammed = outboundIpRewriteCache.contains(cacheKey);
1056
1057         if (action == Action.DELETE && isProgrammed == Boolean.FALSE) {
1058             LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1059                          "is already done",
1060                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1061             return;
1062         }
1063         if (action == Action.ADD && isProgrammed == Boolean.TRUE) {
1064             LOGGER.trace("programOutboundIpRewriteStage1 dpid {} seg {} fixedIpAddress {} floatIp {} action {} " +
1065                          "is already done",
1066                     fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1067             return;
1068         }
1069
1070         Status status = this.programOutboundIpRewriteStage2(fid, action);
1071         if (status.isSuccess()) {
1072             // Update cache
1073             if (action == Action.ADD) {
1074                 outboundIpRewriteCache.add(cacheKey);
1075             } else {
1076                 outboundIpRewriteCache.remove(cacheKey);
1077             }
1078         }
1079     }
1080
1081     private Status programOutboundIpRewriteStage2(FloatIpData fid, Action action) {
1082         Status status;
1083         try {
1084             InetAddress matchSrcAddress = InetAddress.getByName(fid.fixedIpAddress);
1085             InetAddress rewriteSrcAddress = InetAddress.getByName(fid.floatingIpAddress);
1086             status = outboundNatProvider == null ?
1087                     new Status(StatusCode.SUCCESS) :
1088                     outboundNatProvider.programIpRewriteRule(
1089                             fid.dpid, fid.segId, fid.neutronRouterMac, matchSrcAddress, fid.macAddress,
1090                             this.externalRouterMac, rewriteSrcAddress, fid.ofPort, action);
1091         } catch (UnknownHostException e) {
1092             status = new Status(StatusCode.BADREQUEST);
1093         }
1094
1095         if (status.isSuccess()) {
1096             final boolean isSkipped = outboundNatProvider == null;
1097             LOGGER.debug("programOutboundIpRewriteStage2 {} for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1098                             " action {}",
1099                          isSkipped ? "skipped" : "programmed",
1100                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action);
1101         } else {
1102             LOGGER.error("programOutboundIpRewriteStage2 failed for dpid {} seg {} fixedIpAddress {} floatIp {}" +
1103                          " action {} status:{}",
1104                          fid.dpid, fid.segId, fid.fixedIpAddress, fid.floatingIpAddress, action, status);
1105         }
1106         return status;
1107     }
1108
1109     private int getMaskLenFromCidr(String cidr) {
1110         if (cidr == null) {
1111             return 0;
1112         }
1113         String[] splits = cidr.split("/");
1114         if (splits.length != 2) {
1115             return 0;
1116         }
1117
1118         int result;
1119         try {
1120             result = Integer.parseInt(splits[1].trim());
1121         } catch (NumberFormatException nfe) {
1122             result = 0;
1123         }
1124         return result;
1125     }
1126
1127     private Long getDpidForIntegrationBridge(Node node) {
1128         // Check if node is integration bridge; and only then return its dpid
1129         if (southbound.getBridge(node, configurationService.getIntegrationBridgeName()) != null) {
1130             return southbound.getDataPathId(node);
1131         }
1132         return null;
1133     }
1134
1135     /**
1136      * Return String that represents OF port with marker explicitly provided (reverse of MatchUtils:parseExplicitOFPort)
1137      *
1138      * @param ofPort the OF port number
1139      * @return the string with encoded OF port (example format "OFPort|999")
1140      */
1141     public static String encodeExcplicitOFPort(Long ofPort) {
1142         return "OFPort|" + ofPort.toString();
1143     }
1144
1145     @Override
1146     public void setDependencies(BundleContext bundleContext, ServiceReference serviceReference) {
1147         tenantNetworkManager =
1148                 (TenantNetworkManager) ServiceHelper.getGlobalInstance(TenantNetworkManager.class, this);
1149         configurationService =
1150                 (ConfigurationService) ServiceHelper.getGlobalInstance(ConfigurationService.class, this);
1151         arpProvider =
1152                 (ArpProvider) ServiceHelper.getGlobalInstance(ArpProvider.class, this);
1153         inboundNatProvider =
1154                 (InboundNatProvider) ServiceHelper.getGlobalInstance(InboundNatProvider.class, this);
1155         outboundNatProvider =
1156                 (OutboundNatProvider) ServiceHelper.getGlobalInstance(OutboundNatProvider.class, this);
1157         routingProvider =
1158                 (RoutingProvider) ServiceHelper.getGlobalInstance(RoutingProvider.class, this);
1159         l3ForwardingProvider =
1160                 (L3ForwardingProvider) ServiceHelper.getGlobalInstance(L3ForwardingProvider.class, this);
1161         nodeCacheManager =
1162                 (NodeCacheManager) ServiceHelper.getGlobalInstance(NodeCacheManager.class, this);
1163         southbound =
1164                 (Southbound) ServiceHelper.getGlobalInstance(Southbound.class, this);
1165
1166         initL3AdapterMembers();
1167     }
1168
1169     @Override
1170     public void setDependencies(Object impl) {
1171         if (impl instanceof INeutronNetworkCRUD) {
1172             neutronNetworkCache = (INeutronNetworkCRUD)impl;
1173         } else if (impl instanceof INeutronPortCRUD) {
1174             neutronPortCache = (INeutronPortCRUD)impl;
1175         } else if (impl instanceof INeutronSubnetCRUD) {
1176             neutronSubnetCache = (INeutronSubnetCRUD)impl;
1177         } else if (impl instanceof ArpProvider) {
1178             arpProvider = (ArpProvider)impl;
1179         } else if (impl instanceof InboundNatProvider) {
1180             inboundNatProvider = (InboundNatProvider)impl;
1181         } else if (impl instanceof OutboundNatProvider) {
1182             outboundNatProvider = (OutboundNatProvider)impl;
1183         } else if (impl instanceof RoutingProvider) {
1184             routingProvider = (RoutingProvider)impl;
1185         } else if (impl instanceof L3ForwardingProvider) {
1186             l3ForwardingProvider = (L3ForwardingProvider)impl;
1187         }
1188     }
1189 }