openstack/net-virt/src/main/java/org/opendaylight/ovsdb/openstack/netvirt/impl/NeutronL3Adapter.java

   1 /*
   2  * Copyright (C) 2014 Red Hat, Inc.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  *
   8  * Authors : Dave Tucker, Flavio Fernandes
   9  */
  10
  11 package org.opendaylight.ovsdb.openstack.netvirt.impl;
  12
  13 import org.opendaylight.controller.networkconfig.neutron.INeutronNetworkCRUD;
  14 import org.opendaylight.controller.networkconfig.neutron.INeutronPortCRUD;
  15 import org.opendaylight.controller.networkconfig.neutron.INeutronSubnetCRUD;
  16 import org.opendaylight.controller.networkconfig.neutron.NeutronFloatingIP;
  17 import org.opendaylight.controller.networkconfig.neutron.NeutronNetwork;
  18 import org.opendaylight.controller.networkconfig.neutron.NeutronPort;
  19 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter;
  20 import org.opendaylight.controller.networkconfig.neutron.NeutronRouter_Interface;
  21 import org.opendaylight.controller.networkconfig.neutron.NeutronSubnet;
  22 import org.opendaylight.controller.networkconfig.neutron.Neutron_IPs;
  23 import org.opendaylight.controller.sal.core.Node;
  24 import org.opendaylight.controller.sal.utils.HexEncode;
  25 import org.opendaylight.controller.sal.utils.Status;
  26 import org.opendaylight.controller.sal.utils.StatusCode;
  27 import org.opendaylight.ovsdb.lib.notation.Row;
  28 import org.opendaylight.ovsdb.openstack.netvirt.api.MultiTenantAwareRouter;
  29 import org.opendaylight.ovsdb.openstack.netvirt.api.NetworkingProviderManager;
  30 import org.opendaylight.ovsdb.openstack.netvirt.api.TenantNetworkManager;
  31 import org.opendaylight.ovsdb.plugin.api.OvsdbConfigurationService;
  32 import org.opendaylight.ovsdb.plugin.api.OvsdbConnectionService;
  33 import org.opendaylight.ovsdb.schema.openvswitch.Bridge;
  34 import org.opendaylight.ovsdb.openstack.netvirt.api.Action;
  35 import org.opendaylight.ovsdb.openstack.netvirt.api.ArpProvider;
  36 import org.opendaylight.ovsdb.openstack.netvirt.api.InboundNatProvider;
  37 import org.opendaylight.ovsdb.openstack.netvirt.api.L3ForwardingProvider;
  38 import org.opendaylight.ovsdb.openstack.netvirt.api.OutboundNatProvider;
  39 import org.opendaylight.ovsdb.openstack.netvirt.api.RoutingProvider;
  40 import org.opendaylight.ovsdb.schema.openvswitch.Interface;
  41
  42 import com.google.common.base.Preconditions;
  43 import org.slf4j.Logger;
  44 import org.slf4j.LoggerFactory;
  45
  46 import java.net.InetAddress;
  47 import java.net.UnknownHostException;
  48 import java.util.HashMap;
  49 import java.util.HashSet;
  50 import java.util.List;
  51 import java.util.Map;
  52 import java.util.Set;
  53
  54 /**
  55  * Neutron L3 Adapter implements a hub-like adapter for the various Neutron events. Based on
  56  * these events, the abstract router callbacks can be generated to the multi-tenant aware router,
  57  * as well as the multi-tenant router forwarding provider.
  58  */
  59 public class NeutronL3Adapter {
  60
  61     /**
  62      * Logger instance.
  63      */
  64     static final Logger logger = LoggerFactory.getLogger(NeutronL3Adapter.class);
  65
  66     // The implementation for each of these services is resolved by the OSGi Service Manager
  67     private volatile org.opendaylight.ovsdb.openstack.netvirt.api.ConfigurationService configurationService;
  68     private volatile TenantNetworkManager tenantNetworkManager;
  69     private volatile NetworkingProviderManager networkingProviderManager;
  70     private volatile OvsdbConfigurationService ovsdbConfigurationService;
  71     private volatile OvsdbConnectionService connectionService;
  72     private volatile INeutronNetworkCRUD neutronNetworkCache;
  73     private volatile INeutronSubnetCRUD neutronSubnetCache;
  74     private volatile INeutronPortCRUD neutronPortCache;
  75     private volatile MultiTenantAwareRouter multiTenantAwareRouter;
  76     private volatile L3ForwardingProvider l3ForwardingProvider;
  77     private volatile InboundNatProvider inboundNatProvider;
  78     private volatile OutboundNatProvider outboundNatProvider;
  79     private volatile ArpProvider arpProvider;
  80     private volatile RoutingProvider routingProvider;
  81
  82     private Set<String> inboundIpRewriteCache;
  83     private Set<String> outboundIpRewriteCache;
  84     private Set<String> inboundIpRewriteExclusionCache;
  85     private Set<String> outboundIpRewriteExclusionCache;
  86     private Set<String> routerInterfacesCache;
  87     private Set<String> staticArpEntryCache;
  88     private Set<String> defaultRouteCache;
  89     private Map<String, String> networkId2MacCache;
  90
  91     void init() {
  92         this.inboundIpRewriteCache = new HashSet<>();
  93         this.outboundIpRewriteCache = new HashSet<>();
  94         this.inboundIpRewriteExclusionCache = new HashSet<>();
  95         this.outboundIpRewriteExclusionCache = new HashSet<>();
  96         this.routerInterfacesCache = new HashSet<>();
  97         this.staticArpEntryCache = new HashSet<>();
  98         this.defaultRouteCache = new HashSet<>();
  99         this.networkId2MacCache = new HashMap();
 100     }
 101
 102     //
 103     // Callbacks from OVSDB's northbound handlers
 104     //
 105
 106     public void handleNeutronSubnetEvent(final NeutronSubnet subnet, Action action) {
 107         logger.debug("Neutron subnet {} event : {}", action, subnet.toString());
 108
 109         // TODO
 110     }
 111
 112     public void handleNeutronPortEvent(final NeutronPort neutronPort, Action action) {
 113         logger.debug("Neutron port {} event : {}", action, neutronPort.toString());
 114
 115         // TODO
 116     }
 117
 118     public void handleNeutronRouterEvent(final NeutronRouter neutronRouter, Action action) {
 119         logger.debug("Neutron router {} event : {}", action, neutronRouter.toString());
 120
 121         // TODO
 122     }
 123
 124     public void handleNeutronRouterInterfaceEvent(final NeutronRouter neutronRouter,
 125                                                   final NeutronRouter_Interface neutronRouterInterface,
 126                                                   Action action) {
 127         logger.debug(" Router {} interface {} got event {}. Subnet {}",
 128                      neutronRouter.getName(),
 129                      neutronRouterInterface.getPortUUID(),
 130                      action,
 131                      neutronRouterInterface.getSubnetUUID());
 132
 133         this.programFlowsForNeutronRouterInterface(neutronRouterInterface, action == Action.DELETE);
 134     }
 135
 136     public void handleNeutronFloatingIPEvent(final NeutronFloatingIP neutronFloatingIP,
 137                                              Action action) {
 138         logger.debug(" Floating IP {} {}<->{}, network uuid {}", action,
 139                      neutronFloatingIP.getFixedIPAddress(),
 140                      neutronFloatingIP.getFloatingIPAddress(),
 141                      neutronFloatingIP.getFloatingNetworkUUID());
 142
 143         this.programFlowsForFloatingIP(neutronFloatingIP, action == Action.DELETE);
 144     }
 145
 146     public void handleNeutronNetworkEvent(final NeutronNetwork neutronNetwork, Action action) {
 147         logger.debug("neutronNetwork {}: network: {}", action, neutronNetwork);
 148
 149         // TODO
 150     }
 151
 152     //
 153     // Callbacks from OVSDB's southbound handler
 154     //
 155     public void handleInterfaceEvent(final Node node, final Interface intf, final NeutronNetwork neutronNetwork,
 156                                      Action action) {
 157         logger.debug("southbound interface {} node:{} interface:{}, neutronNetwork:{}",
 158                      action, node, intf, neutronNetwork);
 159
 160         // TODO
 161     }
 162
 163     //
 164     // Internal helpers
 165     //
 166     private void programFlowsForNeutronRouterInterface(final NeutronRouter_Interface neutronRouterInterface,
 167                                                        Boolean isDelete) {
 168         Preconditions.checkNotNull(neutronRouterInterface);
 169
 170         final NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
 171         final String macAddress = neutronPort != null ? neutronPort.getMacAddress() : null;
 172         final List<Neutron_IPs> ipList = neutronPort != null ? neutronPort.getFixedIPs() : null;
 173         final NeutronSubnet subnet = neutronSubnetCache.getSubnet(neutronRouterInterface.getSubnetUUID());
 174         final NeutronNetwork neutronNetwork = subnet != null ?
 175                                               neutronNetworkCache.getNetwork(subnet.getNetworkUUID()) : null;
 176         final String providerSegmentationId = neutronNetwork != null ?
 177                                               neutronNetwork.getProviderSegmentationID() : null;
 178         final String gatewayIp = subnet != null ? subnet.getGatewayIP() : null;
 179         final Boolean isExternal = neutronNetwork != null ? neutronNetwork.getRouterExternal() : Boolean.TRUE;
 180         final String cidr = subnet != null ? subnet.getCidr() : null;
 181         final int mask = getMaskLenFromCidr(cidr);
 182
 183         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 184             cidr == null || cidr.isEmpty() ||
 185             macAddress == null || macAddress.isEmpty() ||
 186             ipList == null || ipList.isEmpty()) {
 187             return;  // done: go no further w/out all the info needed...
 188         }
 189
 190         final Action action = isDelete ? Action.DELETE : Action.ADD;
 191
 192         // Keep cache for finding router's mac from network uuid
 193         //
 194         if (isDelete) {
 195             networkId2MacCache.remove(neutronNetwork.getNetworkUUID());
 196         } else {
 197             networkId2MacCache.put(neutronNetwork.getNetworkUUID(), macAddress);
 198         }
 199
 200         List<Node> nodes = connectionService.getNodes();
 201         for (Node node : nodes) {
 202             final Long dpid = getDpid(node);
 203             final Action actionForNode =
 204                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 205                     action : Action.DELETE;
 206
 207             for (Neutron_IPs neutronIP : ipList) {
 208                 final String ipStr = neutronIP.getIpAddress();
 209                 if (ipStr.isEmpty()) continue;
 210                 programRouterInterfaceStage1(node, dpid, providerSegmentationId, macAddress, ipStr, mask, actionForNode);
 211                 programStaticArpStage1(node, dpid, providerSegmentationId, macAddress, ipStr, actionForNode);
 212             }
 213
 214             // Compute action to be programmed. In the case of rewrite exclusions, we must never program rules
 215             // for the external neutron networks.
 216             //
 217             {
 218                 final Action actionForRewriteExclusion = isExternal ? Action.DELETE : actionForNode;
 219                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 220                                                 cidr, actionForRewriteExclusion);
 221                 programIpRewriteExclusionStage1(node, dpid, providerSegmentationId, false /* isInbound */,
 222                                                 cidr, actionForRewriteExclusion);
 223             }
 224
 225             // Default route. For non-external subnets, make sure that there is none configured.
 226             //
 227             if (gatewayIp != null && !gatewayIp.isEmpty()) {
 228                 final Action actionForNodeDefaultRoute =
 229                         isExternal ? actionForNode : Action.DELETE;
 230                 final String defaultGatewayMacAddress = "00:01:02:03:04:05";  // FIXME!
 231                 programDefaultRouteStage1(node, dpid, providerSegmentationId, defaultGatewayMacAddress, gatewayIp,
 232                                           actionForNodeDefaultRoute);
 233             }
 234         }
 235     }
 236
 237     private void programRouterInterfaceStage1(Node node, Long dpid, String providerSegmentationId,
 238                                               String macAddress, String ipStr, int mask,
 239                                               Action actionForNode) {
 240         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 241         // will look at desired action for node.
 242         //
 243         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 244                                 ipStr + "/" + Integer.toString(mask);
 245         final Boolean isProgrammed = routerInterfacesCache.contains(cacheKey);
 246
 247         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE)
 248             return;
 249         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE)
 250             return;
 251
 252         Status status = this.programRouterInterfaceStage2(node, dpid, providerSegmentationId,
 253                                                           macAddress, ipStr, mask, actionForNode);
 254         if (status.isSuccess()) {
 255             // Update cache
 256             if (actionForNode == Action.ADD) {
 257                 // TODO: multiTenantAwareRouter.addInterface(UUID.fromString(tenant), ...);
 258                 routerInterfacesCache.add(cacheKey);
 259             } else {
 260                 // TODO: multiTenantAwareRouter.removeInterface(...);
 261                 routerInterfacesCache.remove(cacheKey);
 262             }
 263         }
 264     }
 265
 266     private Status programRouterInterfaceStage2(Node node, Long dpid, String providerSegmentationId,
 267                                                 String macAddress,
 268                                                 String address, int mask,
 269                                                 Action actionForNode) {
 270         Status status;
 271         try {
 272             InetAddress inetAddress = InetAddress.getByName(address);
 273             status = routingProvider == null ?
 274                      new Status(StatusCode.SUCCESS) :
 275                      routingProvider.programRouterInterface(node, dpid, providerSegmentationId,
 276                                                             macAddress, inetAddress, mask, actionForNode);
 277         } catch (UnknownHostException e) {
 278             status = new Status(StatusCode.BADREQUEST);
 279         }
 280
 281         if (status.isSuccess()) {
 282             logger.debug("ProgramRouterInterface {} for mac:{} addr:{}/{} node:{} action:{}",
 283                          routingProvider == null ? "skipped" : "programmed",
 284                          macAddress, address, mask, node, actionForNode);
 285         } else {
 286             logger.error("ProgramRouterInterface failed for mac:{} addr:{}/{} node:{} action:{} status:{}",
 287                          macAddress, address, mask, node, actionForNode, status);
 288         }
 289         return status;
 290     }
 291
 292     private void programStaticArpStage1(Node node, Long dpid, String providerSegmentationId,
 293                                         String macAddress, String ipStr,
 294                                         Action actionForNode) {
 295         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 296         // will look at desired action for node.
 297         //
 298         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + ipStr;
 299         final Boolean isProgrammed = staticArpEntryCache.contains(cacheKey);
 300
 301         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE)
 302             return;
 303         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE)
 304             return;
 305
 306         Status status = this.programStaticArpStage2(node, dpid, providerSegmentationId,
 307                                                     macAddress, ipStr, actionForNode);
 308         if (status.isSuccess()) {
 309             // Update cache
 310             if (actionForNode == Action.ADD) {
 311                 staticArpEntryCache.add(cacheKey);
 312             } else {
 313                 staticArpEntryCache.remove(cacheKey);
 314             }
 315         }
 316     }
 317
 318     private Status programStaticArpStage2(Node node, Long dpid, String providerSegmentationId,
 319                                                 String macAddress,
 320                                                 String address,
 321                                                 Action actionForNode) {
 322         Status status;
 323         try {
 324             InetAddress inetAddress = InetAddress.getByName(address);
 325             status = arpProvider == null ?
 326                      new Status(StatusCode.SUCCESS) :
 327                      arpProvider.programStaticArpEntry(node, dpid, providerSegmentationId,
 328                                                        macAddress, inetAddress, actionForNode);
 329         } catch (UnknownHostException e) {
 330             status = new Status(StatusCode.BADREQUEST);
 331         }
 332
 333         if (status.isSuccess()) {
 334             logger.debug("ProgramStaticArp {} for mac:{} addr:{} node:{} action:{}",
 335                          arpProvider == null ? "skipped" : "programmed",
 336                          macAddress, address, node, actionForNode);
 337         } else {
 338             logger.error("ProgramStaticArp failed for mac:{} addr:{} node:{} action:{} status:{}",
 339                          macAddress, address, node, actionForNode, status);
 340         }
 341         return status;
 342     }
 343
 344     private void programIpRewriteExclusionStage1(Node node, Long dpid, String providerSegmentationId,
 345                                                  final boolean isInbound, String cidr,
 346                                                  Action actionForRewriteExclusion) {
 347         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 348         // will look at desired action for node.
 349         //
 350         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + cidr;
 351         final Boolean isProgrammed = isInbound ?
 352                                      inboundIpRewriteExclusionCache.contains(cacheKey):
 353                                      outboundIpRewriteExclusionCache.contains(cacheKey);
 354
 355         if (actionForRewriteExclusion == Action.DELETE && isProgrammed == Boolean.FALSE)
 356             return;
 357         if (actionForRewriteExclusion == Action.ADD && isProgrammed == Boolean.TRUE)
 358             return;
 359
 360         Status status = this.programIpRewriteExclusionStage2(node, dpid, providerSegmentationId, cidr,
 361                                                              isInbound, actionForRewriteExclusion);
 362         if (status.isSuccess()) {
 363             // Update cache
 364             if (actionForRewriteExclusion == Action.ADD) {
 365                 if (isInbound) {
 366                     inboundIpRewriteExclusionCache.add(cacheKey);
 367                 } else {
 368                     outboundIpRewriteExclusionCache.add(cacheKey);
 369                 }
 370             } else {
 371                 if (isInbound) {
 372                     inboundIpRewriteExclusionCache.remove(cacheKey);
 373                 } else {
 374                     outboundIpRewriteExclusionCache.remove(cacheKey);
 375                 }
 376             }
 377         }
 378     }
 379
 380     private Status programIpRewriteExclusionStage2(Node node, Long dpid, String providerSegmentationId, String cidr,
 381                                                    final boolean isInbound, Action actionForNode) {
 382         Status status;
 383         if (isInbound) {
 384             status = inboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 385                      inboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 386                                                                   actionForNode);
 387         } else {
 388             status = outboundNatProvider == null ? new Status(StatusCode.SUCCESS) :
 389                      outboundNatProvider.programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr,
 390                                                                    actionForNode);
 391         }
 392
 393         if (status.isSuccess()) {
 394             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 395             logger.debug("IpRewriteExclusion {} {} for cidr:{} node:{} action:{}",
 396                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 397                          cidr, node, actionForNode);
 398         } else {
 399             logger.error("IpRewriteExclusion {} failed for cidr:{} node:{} action:{} status:{}",
 400                          (isInbound ? "inbound" : "outbound"), cidr, node, actionForNode, status);
 401         }
 402         return status;
 403     }
 404
 405     private void programDefaultRouteStage1(Node node, Long dpid, String providerSegmentationId,
 406                                            String defaultGatewayMacAddress, String gatewayIp,
 407                                            Action actionForNodeDefaultRoute) {
 408         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 409         // will look at desired action for node.
 410         //
 411         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" + gatewayIp;
 412         final Boolean isProgrammed = defaultRouteCache.contains(cacheKey);
 413
 414         if (actionForNodeDefaultRoute == Action.DELETE && isProgrammed == Boolean.FALSE)
 415             return;
 416         if (actionForNodeDefaultRoute == Action.ADD && isProgrammed == Boolean.TRUE)
 417             return;
 418
 419         Status status = this.programDefaultRouteStage2(node, dpid, providerSegmentationId,
 420                                                        defaultGatewayMacAddress, gatewayIp, actionForNodeDefaultRoute);
 421         if (status.isSuccess()) {
 422             // Update cache
 423             if (actionForNodeDefaultRoute == Action.ADD) {
 424                 defaultRouteCache.add(cacheKey);
 425             } else {
 426                 defaultRouteCache.remove(cacheKey);
 427             }
 428         }
 429     }
 430
 431     private Status programDefaultRouteStage2(Node node, Long dpid, String providerSegmentationId,
 432                                           String defaultGatewayMacAddress,
 433                                           String gatewayIp,
 434                                           Action actionForNodeDefaultRoute) {
 435         Status status;
 436         try {
 437             InetAddress inetAddress = InetAddress.getByName(gatewayIp);
 438             status = routingProvider == null ?
 439                      new Status(StatusCode.SUCCESS) :
 440                      routingProvider.programDefaultRouteEntry(node, dpid, providerSegmentationId,
 441                                                               defaultGatewayMacAddress, inetAddress,
 442                                                               actionForNodeDefaultRoute);
 443         } catch (UnknownHostException e) {
 444             status = new Status(StatusCode.BADREQUEST);
 445         }
 446
 447         if (status.isSuccess()) {
 448             logger.debug("ProgramDefaultRoute {} for mac:{} gatewayIp:{} node:{} action:{}",
 449                          routingProvider == null ? "skipped" : "programmed",
 450                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute);
 451         } else {
 452             logger.error("ProgramDefaultRoute failed for mac:{} gatewayIp:{} node:{} action:{} status:{}",
 453                          defaultGatewayMacAddress, gatewayIp, node, actionForNodeDefaultRoute, status);
 454         }
 455         return status;
 456     }
 457
 458     private void programFlowsForFloatingIP(final NeutronFloatingIP neutronFloatingIP, Boolean isDelete) {
 459         Preconditions.checkNotNull(neutronFloatingIP);
 460
 461         final String networkUUID = neutronFloatingIP.getFloatingNetworkUUID();
 462         final NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(networkUUID);
 463         final String providerSegmentationId = neutronNetwork != null ?
 464                                               neutronNetwork.getProviderSegmentationID() : null;
 465         final String routerMacAddress = networkId2MacCache.get(networkUUID);
 466         final String fixedIPAddress = neutronFloatingIP.getFixedIPAddress();
 467         final String floatingIpAddress = neutronFloatingIP.getFloatingIPAddress();
 468
 469         if (providerSegmentationId == null || providerSegmentationId.isEmpty() ||
 470             routerMacAddress == null || routerMacAddress.isEmpty() ||
 471             fixedIPAddress == null || fixedIPAddress.isEmpty() ||
 472             floatingIpAddress == null || floatingIpAddress.isEmpty()) {
 473             return;  // done: go no further w/out all the info needed...
 474         }
 475
 476         final Action action = isDelete ? Action.DELETE : Action.ADD;
 477         List<Node> nodes = connectionService.getNodes();
 478         for (Node node : nodes) {
 479             final Long dpid = getDpid(node);
 480             final Action actionForNode =
 481                     tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId) ?
 482                     action : Action.DELETE;
 483
 484             // Rewrite from float to fixed and vice-versa
 485             //
 486             programIpRewriteStage1(node, dpid, providerSegmentationId, true /* isInbound */,
 487                                    floatingIpAddress, fixedIPAddress, actionForNode);
 488             programIpRewriteStage1(node, dpid, providerSegmentationId, false /* isInboubd */,
 489                                    fixedIPAddress, floatingIpAddress, actionForNode);
 490
 491             // Respond to arps for the floating ip address
 492             //
 493             programStaticArpStage1(node, dpid, providerSegmentationId, routerMacAddress, floatingIpAddress,
 494                                    actionForNode);
 495         }
 496     }
 497
 498     private void programIpRewriteStage1(Node node, Long dpid, String providerSegmentationId,
 499                                         final boolean isInbound,
 500                                         String matchAddress, String rewriteAddress,
 501                                         Action actionForNode) {
 502         // Based on the local cache, figure out whether programming needs to occur. To do this, we
 503         // will look at desired action for node.
 504         //
 505         final String cacheKey = node.toString() + ":" + providerSegmentationId + ":" +
 506                                 matchAddress + ":" + rewriteAddress;
 507         final Boolean isProgrammed = isInbound ?
 508                                      inboundIpRewriteCache.contains(cacheKey) :
 509                                      outboundIpRewriteCache.contains(cacheKey);
 510
 511         if (actionForNode == Action.DELETE && isProgrammed == Boolean.FALSE)
 512             return;
 513         if (actionForNode == Action.ADD && isProgrammed == Boolean.TRUE)
 514             return;
 515
 516         Status status = this.programIpRewriteStage2(node, dpid, providerSegmentationId, isInbound,
 517                                                     matchAddress, rewriteAddress, actionForNode);
 518         if (status.isSuccess()) {
 519             // Update cache
 520             if (actionForNode == Action.ADD) {
 521                 if (isInbound) {
 522                     inboundIpRewriteCache.add(cacheKey);
 523                 } else {
 524                     outboundIpRewriteCache.add(cacheKey);
 525                 }
 526             } else {
 527                 if (isInbound) {
 528                     inboundIpRewriteCache.remove(cacheKey);
 529                 } else {
 530                     outboundIpRewriteCache.remove(cacheKey);
 531                 }
 532             }
 533         }
 534     }
 535
 536     private Status programIpRewriteStage2(Node node, Long dpid, String providerSegmentationId,
 537                                           final boolean isInbound,
 538                                           String matchAddress, String rewriteAddress,
 539                                           Action actionForNode) {
 540         Status status;
 541         try {
 542             InetAddress inetMatchAddress = InetAddress.getByName(matchAddress);
 543             InetAddress inetRewriteAddress = InetAddress.getByName(rewriteAddress);
 544             if (isInbound) {
 545                 status = inboundNatProvider == null ?
 546                          new Status(StatusCode.SUCCESS) :
 547                          inboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 548                                                                  inetMatchAddress, inetRewriteAddress, actionForNode);
 549             } else {
 550                 status = outboundNatProvider == null ?
 551                          new Status(StatusCode.SUCCESS) :
 552                          outboundNatProvider.programIpRewriteRule(node, dpid, providerSegmentationId,
 553                                                                   inetMatchAddress, inetRewriteAddress, actionForNode);
 554             }
 555         } catch (UnknownHostException e) {
 556             status = new Status(StatusCode.BADREQUEST);
 557         }
 558
 559         if (status.isSuccess()) {
 560             final boolean isSkipped = isInbound ? inboundNatProvider == null : outboundNatProvider == null;
 561             logger.debug("ProgramIpRewrite {} {} for match:{} rewrite:{} node:{} action:{}",
 562                          (isInbound ? "inbound" : "outbound"), (isSkipped ? "skipped" : "programmed"),
 563                          matchAddress, rewriteAddress, node, actionForNode);
 564         } else {
 565             logger.error("ProgramIpRewrite {} failed for match:{} rewrite:{} node:{} action:{} status:{}",
 566                          (isInbound ? "inbound" : "outbound"),
 567                          matchAddress, rewriteAddress, node, actionForNode, status);
 568         }
 569         return status;
 570     }
 571
 572     //
 573     // More Internals
 574     //
 575
 576     private int getMaskLenFromCidr(String cidr) {
 577         if (cidr == null) return 0;
 578         String[] splits = cidr.split("/");
 579         if (splits.length != 2) return 0;
 580
 581         int result;
 582         try {
 583             result = Integer.parseInt(splits[1].trim());
 584         }
 585         catch (NumberFormatException nfe)
 586         {
 587             result = 0;
 588         }
 589         return result;
 590     }
 591
 592     private Long getDpid (Node node) {
 593         Preconditions.checkNotNull(ovsdbConfigurationService);
 594
 595         String bridgeName = configurationService.getIntegrationBridgeName();
 596         String bridgeUuid = this.getInternalBridgeUUID(node, bridgeName);
 597         if (bridgeUuid == null) {
 598             logger.error("Unable to spot Bridge Identifier for {} in {}", bridgeName, node);
 599             return 0L;
 600         }
 601
 602         try {
 603             Row bridgeRow =  ovsdbConfigurationService
 604                     .getRow(node, ovsdbConfigurationService.getTableName(node, Bridge.class), bridgeUuid);
 605             Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeRow);
 606             Set<String> dpids = bridge.getDatapathIdColumn().getData();
 607             if (dpids == null || dpids.size() == 0) return 0L;
 608             return HexEncode.stringToLong((String) dpids.toArray()[0]);
 609         } catch (Exception e) {
 610             logger.error("Error finding Bridge's OF DPID", e);
 611             return 0L;
 612         }
 613     }
 614
 615     private String getInternalBridgeUUID (Node node, String bridgeName) {
 616         Preconditions.checkNotNull(ovsdbConfigurationService);
 617         try {
 618             Map<String, Row>
 619                     bridgeTable = ovsdbConfigurationService.getRows(node, ovsdbConfigurationService.getTableName(node, Bridge.class));
 620             if (bridgeTable == null) return null;
 621             for (String key : bridgeTable.keySet()) {
 622                 Bridge bridge = ovsdbConfigurationService.getTypedRow(node, Bridge.class, bridgeTable.get(key));
 623                 if (bridge.getName().equals(bridgeName)) return key;
 624             }
 625         } catch (Exception e) {
 626             logger.error("Error getting Bridge Identifier for {} / {}", node, bridgeName, e);
 627         }
 628         return null;
 629     }
 630
 631 }
 632
 633 /*
 634 JUNK
 635
 636         // NeutronPort neutronPort = neutronPortCache.getPort(neutronRouterInterface.getPortUUID());
 637         NeutronSubnet subnet = neutronSubnetCache.getSubnet(neutronRouterInterface.getSubnetUUID());
 638         NeutronNetwork neutronNetwork = neutronNetworkCache.getNetwork(subnet.getNetworkUUID());
 639         String providerSegmentationId = neutronNetwork.getProviderSegmentationID();
 640         Boolean isExternal = neutronNetwork.getRouterExternal();
 641         String cidr = subnet.getCidr();
 642
 643         List<Node> nodes = connectionService.getNodes();
 644         for (Node node : nodes) {
 645             if (tenantNetworkManager.isTenantNetworkPresentInNode(node, providerSegmentationId)) {
 646                 Long dpid = getDpid(node);
 647                 Status status = multiTenantRouterForwardingProvider
 648                         .programIpRewriteExclusion(node, dpid, providerSegmentationId, cidr, action);
 649             }
 650         }
 651 */