opendaylight/samples/simpleforwarding/src/main/java/org/opendaylight/controller/samples/simpleforwarding/internal/SimpleForwardingImpl.java

   1
   2 /*
   3  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   4  *
   5  * This program and the accompanying materials are made available under the
   6  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   7  * and is available at http://www.eclipse.org/legal/epl-v10.html
   8  */
   9
  10 package org.opendaylight.controller.samples.simpleforwarding.internal;
  11
  12 import java.net.InetAddress;
  13 import java.util.ArrayList;
  14 import java.util.EnumSet;
  15 import java.util.HashMap;
  16 import java.util.HashSet;
  17 import java.util.Iterator;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Set;
  22 import java.util.concurrent.ConcurrentHashMap;
  23 import java.util.concurrent.ConcurrentMap;
  24 import java.util.Timer;
  25 import java.util.TimerTask;
  26
  27 import org.opendaylight.controller.clustering.services.CacheConfigException;
  28 import org.opendaylight.controller.clustering.services.CacheExistException;
  29 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
  30 import org.opendaylight.controller.clustering.services.IClusterServices;
  31 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
  32 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  33 import org.opendaylight.controller.hosttracker.IfIptoHost;
  34 import org.opendaylight.controller.hosttracker.IfNewHostNotify;
  35 import org.opendaylight.controller.hosttracker.hostAware.HostNodeConnector;
  36 import org.opendaylight.controller.sal.action.Action;
  37 import org.opendaylight.controller.sal.action.Output;
  38 import org.opendaylight.controller.sal.action.PopVlan;
  39 import org.opendaylight.controller.sal.action.SetDlDst;
  40 import org.opendaylight.controller.sal.action.SetVlanId;
  41 import org.opendaylight.controller.sal.core.Edge;
  42 import org.opendaylight.controller.sal.core.Node;
  43 import org.opendaylight.controller.sal.core.NodeConnector;
  44 import org.opendaylight.controller.sal.core.NodeConnector.NodeConnectorIDType;
  45 import org.opendaylight.controller.sal.core.Path;
  46 import org.opendaylight.controller.sal.core.Property;
  47 import org.opendaylight.controller.sal.core.State;
  48 import org.opendaylight.controller.sal.core.UpdateType;
  49 import org.opendaylight.controller.sal.flowprogrammer.Flow;
  50 import org.opendaylight.controller.sal.match.Match;
  51 import org.opendaylight.controller.sal.match.MatchType;
  52 import org.opendaylight.controller.sal.packet.Ethernet;
  53 import org.opendaylight.controller.sal.packet.IDataPacketService;
  54 import org.opendaylight.controller.sal.packet.IListenDataPacket;
  55 import org.opendaylight.controller.sal.packet.IPv4;
  56 import org.opendaylight.controller.sal.packet.Packet;
  57 import org.opendaylight.controller.sal.packet.PacketResult;
  58 import org.opendaylight.controller.sal.packet.RawPacket;
  59 import org.opendaylight.controller.sal.routing.IListenRoutingUpdates;
  60 import org.opendaylight.controller.sal.routing.IRouting;
  61 import org.opendaylight.controller.sal.utils.EtherTypes;
  62 import org.opendaylight.controller.sal.utils.NetUtils;
  63 import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
  64 import org.opendaylight.controller.sal.utils.Status;
  65 import org.opendaylight.controller.samples.simpleforwarding.HostNodePair;
  66 import org.opendaylight.controller.switchmanager.IInventoryListener;
  67 import org.opendaylight.controller.switchmanager.ISwitchManager;
  68 import org.opendaylight.controller.topologymanager.ITopologyManager;
  69 import org.slf4j.Logger;
  70 import org.slf4j.LoggerFactory;
  71
  72 /**
  73  * This class implements basic L3 forwarding within the managed devices.
  74  * Forwarding is only done within configured subnets.</br>
  75  * <br/>
  76  * The basic flow is that the module listens for new hosts from the
  77  * {@link org.opendaylight.controller.hosttracker.IfIptoHost HostTracker}
  78  * service and on discovering a new host it first calls
  79  * <tt>preparePerHostRules()</tt> to create a set of new rules that must be
  80  * installed in the network. This is done by repeatedly calling
  81  * <tt>updatePerHostRuleInSW()</tt> for each switch in the network. Then it
  82  * installs those rules using <tt>installPerHostRules()</tt>.
  83  */
  84 public class SimpleForwardingImpl implements IfNewHostNotify,
  85         IListenRoutingUpdates, IInventoryListener, IListenDataPacket {
  86     private static Logger log = LoggerFactory.getLogger(SimpleForwardingImpl.class);
  87     private static short DEFAULT_IPSWITCH_PRIORITY = 1;
  88     static final String FORWARDING_RULES_CACHE_NAME = "forwarding.ipswitch.rules";
  89     private IfIptoHost hostTracker;
  90     private IForwardingRulesManager frm;
  91     private ITopologyManager topologyManager;
  92     private IRouting routing;
  93
  94     /**
  95      * The set of all forwarding rules: (host) -> (switch -> flowmod). Note that
  96      * the host includes an attachment point and that while the switch appears
  97      * to be a switch's port, in actuality it is a special port which just
  98      * represents the switch.
  99      */
 100     private ConcurrentMap<HostNodePair, HashMap<NodeConnector, FlowEntry>> rulesDB;
 101     private Map<Node, List<FlowEntry>> tobePrunedPos = new HashMap<Node, List<FlowEntry>>();
 102     private IClusterContainerServices clusterContainerService = null;
 103     private ISwitchManager switchManager;
 104     private IDataPacketService dataPacketService;
 105
 106     /**
 107      * Ip packets that are punted may not have their destination known by hostTracker at the time it
 108      * is presented to SimpleForwardingImpl. Instead of dropping the packet, we will keep it around
 109      * for a 'little' while, to accommodate any transients. See bug 590 for more details.
 110      */
 111     private class PendingPacketData {
 112         private final static byte MAX_AGE = 2;
 113
 114         public final IPv4 pkt;
 115         public final NodeConnector incomingNodeConnector;
 116         private byte age;
 117
 118         public PendingPacketData(IPv4 pkt, NodeConnector incomingNodeConnector) {
 119             this.pkt = pkt;
 120             this.incomingNodeConnector = incomingNodeConnector;
 121             this.age = 0;
 122         }
 123         boolean bumpAgeAndCheckIfTooOld() { return ++age > MAX_AGE; }
 124     }
 125     private static final int MAX_PENDING_PACKET_DESTINATIONS = 64;
 126     private ConcurrentMap<InetAddress, PendingPacketData> pendingPacketDestinations;
 127     private Timer pendingPacketsAgerTimer;
 128
 129     private class PendingPacketsAgerTimerHandler extends TimerTask {
 130         @Override
 131         public void run() {
 132             if (pendingPacketDestinations == null) {
 133                 return;
 134             }
 135             try {
 136                 Iterator<ConcurrentMap.Entry<InetAddress, PendingPacketData>> iterator =
 137                         pendingPacketDestinations.entrySet().iterator();
 138                 while (iterator.hasNext()) {
 139                     ConcurrentHashMap.Entry<InetAddress, PendingPacketData> entry = iterator.next();
 140                     InetAddress dIP = entry.getKey();
 141                     PendingPacketData pendingPacketData = entry.getValue();
 142
 143                     if (pendingPacketData.bumpAgeAndCheckIfTooOld()) {
 144                         iterator.remove(); // safe to remove while iterating...
 145                         log.debug("Pending packet for {} has been aged out", dIP);
 146                     } else {
 147                         /** Replace the entry for a key only if currently mapped to some value.
 148                          * This will protect the concurrent map against a race where this thread
 149                          * would be re-adding an entry that just got taken out.
 150                          */
 151                         pendingPacketDestinations.replace(dIP, pendingPacketData);
 152                     }
 153                 }
 154             } catch (IllegalStateException e) {
 155                 log.warn("IllegalStateException Received by PendingPacketsAgerTimerHandler from: {}",
 156                         e.getMessage());
 157             }
 158         }
 159     }
 160
 161     /**
 162      * Add punted packet to pendingPackets
 163      */
 164     private void addToPendingPackets(InetAddress dIP, IPv4 pkt, NodeConnector incomingNodeConnector) {
 165         if (pendingPacketDestinations.size() >= MAX_PENDING_PACKET_DESTINATIONS) {
 166             log.info("Will not pend packet for {}: Too many destinations", dIP);
 167             return;
 168         }
 169
 170         /** TODO: The current implementation allows for up to 1 pending packet per InetAddress.
 171          * This limitation is done for sake of simplicity. A potential enhancement could be to use a
 172          * ConcurrentMultiMap instead of ConcurrentMap.
 173          */
 174         if (pendingPacketDestinations.containsKey(dIP)) {
 175             log.trace("Will not pend packet for {}: Already have a packet pending", dIP);
 176             return;
 177         }
 178
 179         PendingPacketData pendingPacketData = new PendingPacketData(pkt, incomingNodeConnector);
 180         pendingPacketDestinations.put(dIP, pendingPacketData);
 181         log.debug("Pending packet for {}", dIP);
 182     }
 183
 184     /**
 185      * Send punted packet to given destination. This is invoked when there is a certain level of
 186      * hope that the destination is known by hostTracker.
 187      */
 188     private void sendPendingPacket(InetAddress dIP) {
 189         PendingPacketData pendingPacketData = pendingPacketDestinations.get(dIP);
 190         if (pendingPacketData != null) {
 191             handlePuntedIPPacket(pendingPacketData.pkt, pendingPacketData.incomingNodeConnector, false);
 192             log.trace("Packet for {} is no longer pending", dIP);
 193             pendingPacketDestinations.remove(dIP);
 194         }
 195     }
 196
 197     /**
 198      * Return codes from the programming of the perHost rules in HW
 199      */
 200     public enum RulesProgrammingReturnCode {
 201         SUCCESS, FAILED_FEW_SWITCHES, FAILED_ALL_SWITCHES, FAILED_WRONG_PARAMS
 202     }
 203     public void setDataPacketService(IDataPacketService s) {
 204         log.debug("Setting dataPacketService");
 205         this.dataPacketService = s;
 206     }
 207
 208     public void unsetDataPacketService(IDataPacketService s) {
 209         if (this.dataPacketService == s) {
 210             this.dataPacketService = null;
 211         }
 212     }
 213
 214     public void setRouting(IRouting routing) {
 215         log.debug("Setting routing");
 216         this.routing = routing;
 217     }
 218
 219     public void unsetRouting(IRouting routing) {
 220         if (this.routing == routing) {
 221             this.routing = null;
 222         }
 223     }
 224
 225     public void setTopologyManager(ITopologyManager topologyManager) {
 226         log.debug("Setting topologyManager");
 227         this.topologyManager = topologyManager;
 228     }
 229
 230     public void unsetTopologyManager(ITopologyManager topologyManager) {
 231         if (this.topologyManager == topologyManager) {
 232             this.topologyManager = null;
 233         }
 234     }
 235
 236     public void setHostTracker(IfIptoHost hostTracker) {
 237         log.debug("Setting HostTracker");
 238         this.hostTracker = hostTracker;
 239     }
 240
 241     public void setForwardingRulesManager(
 242             IForwardingRulesManager forwardingRulesManager) {
 243         log.debug("Setting ForwardingRulesManager");
 244         this.frm = forwardingRulesManager;
 245     }
 246
 247     public void unsetHostTracker(IfIptoHost hostTracker) {
 248         if (this.hostTracker == hostTracker) {
 249             this.hostTracker = null;
 250         }
 251     }
 252
 253     public void unsetForwardingRulesManager(
 254             IForwardingRulesManager forwardingRulesManager) {
 255         if (this.frm == forwardingRulesManager) {
 256             this.frm = null;
 257         }
 258     }
 259
 260     /**
 261      * Function called when the bundle gets activated
 262      *
 263      */
 264     public void startUp() {
 265         allocateCaches();
 266         retrieveCaches();
 267         nonClusterObjectCreate();
 268     }
 269
 270     public void nonClusterObjectCreate() {
 271         pendingPacketDestinations = new ConcurrentHashMap<InetAddress, PendingPacketData>();
 272
 273         /* Pending Packets Ager Timer to go off every 6 seconds to implement pending packet aging */
 274         pendingPacketsAgerTimer = new Timer();
 275         pendingPacketsAgerTimer.schedule(new PendingPacketsAgerTimerHandler(), 6000, 6000);
 276     }
 277
 278     /**
 279      * Function called when the bundle gets stopped
 280      *
 281      */
 282     public void shutDown() {
 283         log.debug("Destroy all the host Rules given we are shutting down");
 284         uninstallPerHostRules();
 285         destroyCaches();
 286     }
 287
 288     private void allocateCaches() {
 289         if (this.clusterContainerService == null) {
 290             log.trace("un-initialized clusterContainerService, can't create cache");
 291             return;
 292         }
 293
 294         try {
 295             clusterContainerService.createCache(FORWARDING_RULES_CACHE_NAME,
 296                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
 297         } catch (CacheExistException cee) {
 298             log.error("\nCache already exists - destroy and recreate if needed");
 299         } catch (CacheConfigException cce) {
 300             log.error("\nCache configuration invalid - check cache mode");
 301         }
 302     }
 303
 304     @SuppressWarnings({ "unchecked" })
 305     private void retrieveCaches() {
 306         if (this.clusterContainerService == null) {
 307             log.trace("un-initialized clusterContainerService, can't retrieve cache");
 308             return;
 309         }
 310
 311         rulesDB = (ConcurrentMap<HostNodePair, HashMap<NodeConnector, FlowEntry>>) clusterContainerService
 312                 .getCache(FORWARDING_RULES_CACHE_NAME);
 313         if (rulesDB == null) {
 314             log.error("\nFailed to get rulesDB handle");
 315         }
 316     }
 317
 318     private void destroyCaches() {
 319         if (this.clusterContainerService == null) {
 320             log.trace("un-initialized clusterContainerService, can't destroy cache");
 321             return;
 322         }
 323
 324         clusterContainerService.destroyCache(FORWARDING_RULES_CACHE_NAME);
 325     }
 326
 327     /**
 328      * Populates <tt>rulesDB</tt> with rules specifying how to reach
 329      * <tt>host</tt> from <tt>currNode</tt> assuming that:
 330      * <ul>
 331      * <li><tt>host</tt> is attached to <tt>rootNode</tt>
 332      * <li><tt>link</tt> is the next part of the path to reach <tt>rootNode</tt>
 333      * from <tt>currNode</tt>
 334      * <li><tt>rulesDB.get(key)</tt> represents the list of rules stored about
 335      * <tt>host</tt> at <tt>currNode</tt>
 336      * </ul>
 337      *
 338      * @param host
 339      *            The host to be reached.
 340      * @param currNode
 341      *            The current node being processed.
 342      * @param rootNode
 343      *            The node to be reached. Really, the switch which host is
 344      *            attached to.
 345      * @param link
 346      *            The link to follow from curNode to get to rootNode
 347      * @param key
 348      *            The key to store computed rules at in the rulesDB. For now,
 349      *            this is a {@link HostNodePair} of host and currNode.
 350      */
 351     private void updatePerHostRuleInSW(HostNodeConnector host, Node currNode,
 352             Node rootNode, Edge link, HostNodePair key) {
 353
 354         // only the link parameter is optional
 355         if (host == null || key == null || currNode == null || rootNode == null) {
 356             return;
 357         }
 358
 359         Set<NodeConnector> ports = new HashSet<NodeConnector>();
 360         // add a special port of type ALL and port 0 to represent the node
 361         // without specifying a port on that node
 362         ports.add(NodeConnectorCreator.createNodeConnector(
 363                 NodeConnectorIDType.ALL, NodeConnector.SPECIALNODECONNECTORID,
 364                 currNode));
 365
 366         HashMap<NodeConnector, FlowEntry> pos = this.rulesDB.get(key);
 367         if (pos == null) {
 368             pos = new HashMap<NodeConnector, FlowEntry>();
 369         }
 370
 371         for (NodeConnector inPort : ports) {
 372             // skip the port connected to the target host
 373             if (currNode.equals(rootNode)
 374                     && (host.getnodeConnector().equals(inPort))) {
 375                 continue;
 376             }
 377
 378             // remove the current rule, if any
 379             FlowEntry removed_po = pos.remove(inPort);
 380             Match match = new Match();
 381             List<Action> actions = new ArrayList<Action>();
 382
 383             // IP destination based forwarding on /32 entries only!
 384             match.setField(MatchType.DL_TYPE, EtherTypes.IPv4.shortValue());
 385             match.setField(MatchType.NW_DST, host.getNetworkAddress());
 386
 387             /* Action for the policy is to forward to a port except on the
 388              * switch where the host sits, which is to rewrite also the MAC
 389              * and to forward on the Host port */
 390             NodeConnector outPort = null;
 391
 392             if (currNode.equals(rootNode)) {
 393                 /* If we're at the root node, then rewrite the DL addr and
 394                  * possibly pop the VLAN tag. This allows for MAC rewriting
 395                  * in the core of the network assuming we can uniquely ID
 396                  * packets based on IP address. */
 397
 398                 outPort = host.getnodeConnector();
 399                 if (inPort.equals(outPort)) {
 400                     // TODO: isn't this code skipped already by the above continue?
 401                     // skip the host port
 402                     continue;
 403                 }
 404                 actions.add(new SetDlDst(host.getDataLayerAddressBytes()));
 405
 406                 if (!inPort.getType().equals(NodeConnectorIDType.ALL)) {
 407                     // Container mode: at the destination switch, we need to strip out the tag (VLAN)
 408                     actions.add(new PopVlan());
 409                 }
 410             } else {
 411                 // currNode is NOT the rootNode, find the next hop and create a rule
 412                 if (link != null) {
 413                     outPort = link.getTailNodeConnector();
 414                     if (inPort.equals(outPort)) {
 415                         // skip the outgoing port
 416                         continue;
 417                     }
 418
 419                     // If outPort is network link, add VLAN tag
 420                     if (topologyManager.isInternal(outPort)) {
 421                         log.debug("outPort {}/{} is internal uplink port",
 422                                 currNode, outPort);
 423                     } else {
 424                         log.debug("outPort {}/{} is host facing port",
 425                                 currNode, outPort);
 426                     }
 427
 428                     if ((!inPort.getType().equals(NodeConnectorIDType.ALL))
 429                         && (topologyManager.isInternal(outPort))) {
 430                         Node nextNode = link.getHeadNodeConnector()
 431                                             .getNode();
 432                         // TODO: Replace this with SAL equivalent
 433                         //short tag = container.getTag((Long)nextNode.getNodeID());
 434                         short tag = 0;
 435                         if (tag != 0) {
 436                             log.debug("adding SET_VLAN {} for traffic " +
 437                                     "leaving {}/{} toward switch {}",
 438                                     new Object[] { tag, currNode, outPort,
 439                                     nextNode});
 440                             actions.add(new SetVlanId(tag));
 441                         } else {
 442                             log.debug("No tag assigned to switch {}", nextNode);
 443                         }
 444                     }
 445                 }
 446             }
 447             if (outPort != null) {
 448                 actions.add(new Output(outPort));
 449             }
 450             if (!inPort.getType().equals(NodeConnectorIDType.ALL)) {
 451                 // include input port in the flow match field
 452                 match.setField(MatchType.IN_PORT, inPort);
 453
 454                 if (topologyManager.isInternal(inPort)) {
 455                     log.debug("inPort {}/{} is internal uplink port", currNode,
 456                             inPort);
 457                 } else {
 458                     log.debug("inPort {}/{} is host facing port", currNode,
 459                             inPort);
 460                 }
 461
 462                 // for incoming network link; if the VLAN tag is defined, include it for incoming flow matching
 463                 if (topologyManager.isInternal(inPort)) {
 464                     // TODO: Replace this with SAL equivalent
 465                     //short tag = container.getTag((Long)currNode.getNodeID());
 466                     short tag = 0;
 467                     if (tag != 0) {
 468                         log.debug("adding MATCH VLAN {} for traffic entering" +
 469                                 "  {}/{}",
 470                                 new Object[] {tag, currNode, inPort});
 471                         match.setField(MatchType.DL_VLAN, tag);
 472                     } else {
 473                         log.debug("No tag assigned to switch {}", currNode);
 474                     }
 475                 }
 476             }
 477             // Make sure the priority for IP switch entries is
 478             // set to a level just above default drop entries
 479             Flow flow = new Flow(match, actions);
 480             flow.setIdleTimeout((short) 0);
 481             flow.setHardTimeout((short) 0);
 482             flow.setPriority(DEFAULT_IPSWITCH_PRIORITY);
 483
 484             String policyName = host.getNetworkAddress().getHostAddress()
 485                     + "/32";
 486             String flowName = "["
 487                     + (!inPort.getType().equals(NodeConnectorIDType.ALL) ?
 488                        (inPort.getID()).toString()
 489                        + "," : "")
 490                     + host.getNetworkAddress().getHostAddress() + "/32 on N "
 491                     + currNode + "]";
 492             FlowEntry po = new FlowEntry(policyName, flowName, flow, currNode);
 493
 494             /* Now save the rule in the DB rule, so on updates from topology we
 495              * can selectively */
 496             pos.put(inPort, po);
 497             this.rulesDB.put(key, pos);
 498             if (!inPort.getType().equals(NodeConnectorIDType.ALL)) {
 499                 log.debug("Adding Match(inPort = {} , DIP = {})" +
 500                         " Action(outPort= {}) to node {}",
 501                         new Object[] { inPort,
 502                         host.getNetworkAddress().getHostAddress(),
 503                         outPort, currNode});
 504                 if ((removed_po != null)
 505                         && (!po.getFlow().getMatch().equals(
 506                                 removed_po.getFlow().getMatch()))) {
 507                     log.debug("Old Flow match: {}, New Flow match: {}",
 508                             removed_po.getFlow().getMatch(), po.getFlow()
 509                                     .getMatch());
 510                     addTobePrunedPolicy(currNode, removed_po, po);
 511                 }
 512
 513             } else {
 514                 log.debug("Adding policyMatch(DIP = {}) Action(outPort= {}) " +
 515                         "to node {}", new Object[] {
 516                         host.getNetworkAddress().getHostAddress(), outPort,
 517                         currNode});
 518             }
 519         }
 520     }
 521
 522     /**
 523      * Calculate the per-Host rules to be installed in the rulesDB,
 524      * and that will later on be installed in HW, this routine will
 525      * implicitly calculate the shortest path tree among the switch
 526      * to which the host is attached and all the other switches in the
 527      * network and will automatically create all the rules that allow
 528      * a /32 destination IP based forwarding, as in traditional IP
 529      * networks.
 530      *
 531      * @param host Host for which we are going to prepare the rules in the rulesDB
 532      *
 533      * @return A set of switches touched by the calculation
 534      */
 535     private Set<Node> preparePerHostRules(HostNodeConnector host) {
 536         if (host == null) {
 537             return null;
 538         }
 539
 540         //TODO: race condition! unset* functions can make these null.
 541         if (this.routing == null) {
 542             return null;
 543         }
 544         if (this.switchManager == null) {
 545             return null;
 546         }
 547         if (this.rulesDB == null) {
 548             return null;
 549         }
 550
 551         Node rootNode = host.getnodeconnectorNode();
 552         Set<Node> nodes = this.switchManager.getNodes();
 553         Set<Node> switchesToProgram = new HashSet<Node>();
 554         HostNodePair key;
 555         HashMap<NodeConnector, FlowEntry> pos;
 556         FlowEntry po;
 557
 558         // for all nodes in the system
 559         for (Node node : nodes) {
 560             if (node.equals(rootNode)) {
 561                 // We skip it because for the node with host attached
 562                 // we will process in every case even if there are no
 563                 // routes
 564                 continue;
 565             }
 566             List<Edge> links;
 567             Path res = this.routing.getRoute(node, rootNode);
 568             if ((res == null) || ((links = res.getEdges()) == null)) {
 569                 // No route from node to rootNode can be found, back out any
 570                 // existing forwarding rules if they exist.
 571                 log.debug("NO Route/Path between SW[{}] --> SW[{}] cleaning " +
 572                         "potentially existing entries", node, rootNode);
 573                 key = new HostNodePair(host, node);
 574                 pos = this.rulesDB.get(key);
 575                 if (pos != null) {
 576                     for (Map.Entry<NodeConnector, FlowEntry> e : pos.entrySet()) {
 577                         po = e.getValue();
 578                         if (po != null) {
 579                             // uninstall any existing rules we put in the
 580                             // ForwardingRulesManager
 581                             this.frm.uninstallFlowEntry(po);
 582                         }
 583                     }
 584                     this.rulesDB.remove(key);
 585                 }
 586                 continue;
 587             }
 588
 589             log.debug("Route between SW[{}] --> SW[{}]", node, rootNode);
 590             Node currNode = node;
 591             key = new HostNodePair(host, currNode);
 592
 593             // for each link in the route from here to there
 594             for (Edge link : links) {
 595                 if (link == null) {
 596                     log.error("Could not retrieve the Link");
 597                     // TODO: should we keep going?
 598                     continue;
 599                 }
 600
 601                 log.debug(link.toString());
 602
 603                 // Index all the switches to be programmed
 604                 updatePerHostRuleInSW(host, currNode, rootNode, link, key);
 605                 if ((this.rulesDB.get(key)) != null) {
 606                     /* Calling updatePerHostRuleInSW() doesn't guarantee that
 607                      * rules will be added in currNode (e.g, there is only one
 608                      * link from currNode to rootNode This check makes sure that
 609                      * there are some rules in the rulesDB for the given key
 610                      * prior to adding switch to switchesToProgram
 611                      */
 612                     switchesToProgram.add(currNode);
 613                 }
 614                 currNode = link.getHeadNodeConnector().getNode();
 615                 key = new HostNodePair(host, currNode);
 616             }
 617         }
 618
 619         // This rule will be added no matter if any topology is built
 620         // or no, it serve as a way to handle the case of a node with
 621         // multiple hosts attached to it but not yet connected to the
 622         // rest of the world
 623         switchesToProgram.add(rootNode);
 624         updatePerHostRuleInSW(host, rootNode, rootNode, null,
 625                               new HostNodePair(host, rootNode));
 626
 627         //      log.debug("Getting out at the end!");
 628         return switchesToProgram;
 629     }
 630
 631     /**
 632      * Calculate the per-Host rules to be installed in the rulesDB
 633      * from a specific switch when a host facing port comes up.
 634      * These rules will later on be installed in HW. This routine
 635      * will implicitly calculate the shortest path from the switch
 636      * where the port has come up to the switch where host is ,
 637      * attached and will automatically create all the rules that allow
 638      * a /32 destination IP based forwarding, as in traditional IP
 639      * networks.
 640      *
 641      * @param host Host for which we are going to prepare the rules in the rulesDB
 642      * @param swId Switch ID where the port has come up
 643      *
 644      * @return A set of switches touched by the calculation
 645      */
 646     private Set<Node> preparePerHostPerSwitchRules(HostNodeConnector host,
 647             Node node, NodeConnector swport) {
 648         if ((host == null) || (node == null)) {
 649             return null;
 650         }
 651         if (this.routing == null) {
 652             return null;
 653         }
 654         if (this.switchManager == null) {
 655             return null;
 656         }
 657         if (this.rulesDB == null) {
 658             return null;
 659         }
 660
 661         Node rootNode = host.getnodeconnectorNode();
 662         Set<Node> switchesToProgram = new HashSet<Node>();
 663         HostNodePair key;
 664         Map<NodeConnector, FlowEntry> pos;
 665         FlowEntry po;
 666         List<Edge> links;
 667
 668         Path res = this.routing.getRoute(node, rootNode);
 669         if ((res == null) || ((links = res.getEdges()) == null)) {
 670             // the routing service doesn't know how to get there from here
 671             log.debug("NO Route/Path between SW[{}] --> SW[{}] cleaning " +
 672                     "potentially existing entries", node, rootNode);
 673             key = new HostNodePair(host, node);
 674             pos = this.rulesDB.get(key);
 675             if (pos != null) {
 676                 for (Map.Entry<NodeConnector, FlowEntry> e : pos.entrySet()) {
 677                     po = e.getValue();
 678                     if (po != null) {
 679                         //Uninstall the policy
 680                         this.frm.uninstallFlowEntry(po);
 681                     }
 682                 }
 683                 this.rulesDB.remove(key);
 684             }
 685             return null;
 686         }
 687
 688         log.debug("Route between SW[{}] --> SW[{}]", node, rootNode);
 689         Integer curr;
 690         Node currNode = node;
 691         key = new HostNodePair(host, currNode);
 692         Edge link;
 693         for (curr = 0; curr < links.size(); curr++) {
 694             link = links.get(curr);
 695             if (link == null) {
 696                 log.error("Could not retrieve the Link");
 697                 continue;
 698             }
 699
 700             log.debug("Link [{}/{}] --> [{}/{}]", new Object[] {
 701                     currNode, link.getHeadNodeConnector(),
 702                     link.getHeadNodeConnector().getNode(),
 703                     link.getTailNodeConnector()});
 704
 705             // Index all the switches to be programmed
 706             switchesToProgram.add(currNode);
 707             updatePerHostRuleInSW(host, currNode, rootNode, link, key);
 708             break; // come out of the loop for port up case, interested only in programming one switch
 709         }
 710
 711         // This rule will be added no matter if any topology is built
 712         // or no, it serve as a way to handle the case of a node with
 713         // multiple hosts attached to it but not yet connected to the
 714         // rest of the world
 715         // switchesToProgram.add(rootNode);
 716         //updatePerHostRuleInSW(host, rootNode,
 717         //                                        rootNode, null,
 718         //                                        new HostNodePair(host, rootNode),ports);
 719
 720         //      log.debug("Getting out at the end!");
 721         return switchesToProgram;
 722     }
 723
 724     /**
 725      * Routine that fetch the per-Host rules from the rulesDB and
 726      * install in HW, the one having the same match rules will be
 727      * overwritten silently.
 728      *
 729      * @param host host for which we want to install in HW the per-Host rules
 730      * @param switchesToProgram list of switches to be programmed in
 731      * HW, usually are them all, but better to be explicit, that list
 732      * may change with time based on new switch addition/removal
 733      *
 734      * @return a return code that convey the programming status of the HW
 735      */
 736     private RulesProgrammingReturnCode installPerHostRules(
 737             HostNodeConnector host, Set<Node> switchesToProgram) {
 738         RulesProgrammingReturnCode retCode = RulesProgrammingReturnCode.SUCCESS;
 739         if (host == null || switchesToProgram == null) {
 740             return RulesProgrammingReturnCode.FAILED_WRONG_PARAMS;
 741         }
 742         Map<NodeConnector, FlowEntry> pos;
 743         FlowEntry po;
 744         // Now program every single switch
 745         log.debug("Inside installPerHostRules");
 746         for (Node swId : switchesToProgram) {
 747             HostNodePair key = new HostNodePair(host, swId);
 748             pos = this.rulesDB.get(key);
 749             if (pos == null) {
 750                 continue;
 751             }
 752             for (Map.Entry<NodeConnector, FlowEntry> e : pos.entrySet()) {
 753                 po = e.getValue();
 754                 if (po != null) {
 755                     // Populate the Policy field now
 756                     Status poStatus = this.frm.modifyOrAddFlowEntry(po);
 757                     if (!poStatus.isSuccess()) {
 758                         log.error("Failed to install policy: "
 759                                 + po.getGroupName() + " ("
 760                                 + poStatus.getDescription() + ")");
 761
 762                         retCode = RulesProgrammingReturnCode.FAILED_FEW_SWITCHES;
 763                         // Remove the entry from the DB, it was not installed!
 764                         this.rulesDB.remove(key);
 765                     } else {
 766                         log.debug("Successfully installed policy "
 767                                 + po.toString() + " on switch " + swId);
 768                     }
 769                 } else {
 770                     log.error("Cannot find a policy for SW:({}) Host: ({})",
 771                               swId, host);
 772                     /* // Now dump every single rule */
 773                     /* for (HostNodePair dumpkey : this.rulesDB.keySet()) { */
 774                     /*  po = this.rulesDB.get(dumpkey); */
 775                     /*  log.debug("Dumping entry H{" + dumpkey.getHost() + "} S{" + dumpkey.getSwitchId() + "} = {" + (po == null ? "null policy" : po)); */
 776                     /* } */
 777                 }
 778             }
 779         }
 780         log.debug("Leaving installPerHostRules");
 781         return retCode;
 782     }
 783
 784     /**
 785      * Cleanup all the host rules for a given host
 786      *
 787      * @param host Host for which the host rules need to be cleaned
 788      * up, the host could be null in that case it match all the hosts
 789      *
 790      * @return a return code that convey the programming status of the HW
 791      */
 792     private RulesProgrammingReturnCode uninstallPerHostRules(HostNodeConnector host) {
 793         RulesProgrammingReturnCode retCode = RulesProgrammingReturnCode.SUCCESS;
 794         Map<NodeConnector, FlowEntry> pos;
 795         FlowEntry po;
 796         // Now program every single switch
 797         for (HostNodePair key : this.rulesDB.keySet()) {
 798             if (host == null || key.getHost().equals(host)) {
 799                 pos = this.rulesDB.get(key);
 800                 for (Map.Entry<NodeConnector, FlowEntry> e : pos.entrySet()) {
 801                     po = e.getValue();
 802                     if (po != null) {
 803                         // Uninstall the policy
 804                         this.frm.uninstallFlowEntry(po);
 805                     }
 806                 }
 807                 this.rulesDB.remove(key);
 808             }
 809         }
 810         return retCode;
 811     }
 812
 813     /**
 814      * Cleanup all the host rules for a given node, triggered when the
 815      * switch disconnects, so there is no reason for Hw cleanup
 816      * because it's disconnected anyhow
 817      * TBD - Revisit above stmt in light of CSCus88743
 818      * @param targetNode Node for which we want to do cleanup
 819      *
 820      */
 821     private void uninstallPerNodeRules(Node targetNode) {
 822         //RulesProgrammingReturnCode retCode = RulesProgrammingReturnCode.SUCCESS;
 823         Map<NodeConnector, FlowEntry> pos;
 824         FlowEntry po;
 825
 826         // Now program every single switch
 827         for (HostNodePair key : this.rulesDB.keySet()) {
 828             Node node = key.getNode();
 829             if (targetNode == null || node.equals(targetNode)) {
 830                 log.debug("Work on {} host {}", node, key.getHost());
 831                 pos = this.rulesDB.get(key);
 832                 for (Map.Entry<NodeConnector, FlowEntry> e : pos.entrySet()) {
 833                     po = e.getValue();
 834                     if (po != null) {
 835                         // Uninstall the policy
 836                         this.frm.uninstallFlowEntry(po);
 837                     }
 838                 }
 839                 log.debug("Remove {}", key);
 840                 this.rulesDB.remove(key);
 841             }
 842         }
 843     }
 844
 845     /**
 846      * Cleanup all the host rules currently present in the rulesDB
 847      *
 848      * @return a return code that convey the programming status of the HW
 849      */
 850     private RulesProgrammingReturnCode uninstallPerHostRules() {
 851         return uninstallPerHostRules(null);
 852     }
 853
 854     @Override
 855     public void recalculateDone() {
 856         if (this.hostTracker == null) {
 857             //Not yet ready to process all the updates
 858             //TODO: we should make sure that this call is executed eventually
 859             return;
 860         }
 861         Set<HostNodeConnector> allHosts = this.hostTracker.getAllHosts();
 862         for (HostNodeConnector host : allHosts) {
 863             Set<Node> switches = preparePerHostRules(host);
 864             if (switches != null) {
 865                 // This will refresh existing rules, by overwriting
 866                 // the previous ones
 867                 installPerHostRules(host, switches);
 868                 pruneExcessRules(switches);
 869             }
 870         }
 871     }
 872
 873     void addTobePrunedPolicy(Node swId, FlowEntry po, FlowEntry new_po) {
 874         List<FlowEntry> pl = tobePrunedPos.get(swId);
 875         if (pl == null) {
 876             pl = new LinkedList<FlowEntry>();
 877             tobePrunedPos.put(swId, pl);
 878         }
 879         pl.add(po);
 880         log.debug("Adding Pruned Policy for SwId: {}", swId);
 881         log.debug("Old Policy: {}", po);
 882         log.debug("New Policy: {}", new_po);
 883     }
 884
 885     private void pruneExcessRules(Set<Node> switches) {
 886         for (Node swId : switches) {
 887             List<FlowEntry> pl = tobePrunedPos.get(swId);
 888             if (pl != null) {
 889                 log.debug("Policies for Switch: {} in the list to be deleted: {}", swId, pl);
 890                 Iterator<FlowEntry> plIter = pl.iterator();
 891                 //for (Policy po: pl) {
 892                 while (plIter.hasNext()) {
 893                     FlowEntry po = plIter.next();
 894                     log.error("Removing Policy, Switch: {} Policy: {}", swId, po);
 895                     this.frm.uninstallFlowEntry(po);
 896                     plIter.remove();
 897                 }
 898             }
 899             // tobePrunedPos.remove(swId);
 900         }
 901     }
 902
 903     /**
 904      * A Host facing port has come up in a container. Add rules on the switch where this
 905      * port has come up for all the known hosts to the controller.
 906      * @param swId switch id of the port where port came up
 907      * @param swPort port which came up
 908      */
 909     private void updateRulesforHIFup(Node node, NodeConnector swPort) {
 910         if (this.hostTracker == null) {
 911             //Not yet ready to process all the updates
 912             return;
 913         }
 914         log.debug("Host Facing Port in a container came up, install the rules for all hosts from this port !");
 915         Set<HostNodeConnector> allHosts = this.hostTracker.getAllHosts();
 916         for (HostNodeConnector host : allHosts) {
 917             if (node.equals(host.getnodeconnectorNode())) {
 918                 /*
 919                  * This host resides behind the same switch and port for which a port up
 920                  * message is received. Ideally this should not happen, but if it does,
 921                  * don't program any rules for this host
 922                  */
 923                 continue;
 924             }
 925             Set<Node> switches = preparePerHostPerSwitchRules(host, node,
 926                     swPort);
 927             if (switches != null) {
 928                 // This will refresh existing rules, by overwriting
 929                 // the previous ones
 930                 installPerHostRules(host, switches);
 931             }
 932         }
 933
 934     }
 935
 936     @Override
 937     public void notifyHTClient(HostNodeConnector host) {
 938         if (host == null) {
 939             return;
 940         }
 941         Set<Node> switches = preparePerHostRules(host);
 942         if (switches != null) {
 943             installPerHostRules(host, switches);
 944
 945             // Green light for sending pending packet to this host. Safe to call if there are none.
 946             sendPendingPacket(host.getNetworkAddress());
 947         }
 948     }
 949
 950     @Override
 951     public void notifyHTClientHostRemoved(HostNodeConnector host) {
 952         if (host == null) {
 953             return;
 954         }
 955         uninstallPerHostRules(host);
 956     }
 957
 958     @Override
 959     public void notifyNode(Node node, UpdateType type,
 960             Map<String, Property> propMap) {
 961         if (node == null) {
 962             return;
 963         }
 964
 965         switch (type) {
 966         case REMOVED:
 967             log.debug("Node {} gone, doing a cleanup", node);
 968             uninstallPerNodeRules(node);
 969             break;
 970         default:
 971             break;
 972         }
 973     }
 974
 975     @Override
 976     public void notifyNodeConnector(NodeConnector nodeConnector,
 977             UpdateType type, Map<String, Property> propMap) {
 978         if (nodeConnector == null) {
 979             return;
 980         }
 981
 982         boolean up = false;
 983         switch (type) {
 984         case ADDED:
 985             up = true;
 986             break;
 987         case REMOVED:
 988             break;
 989         case CHANGED:
 990             State state = (State) propMap.get(State.StatePropName);
 991             if ((state != null) && (state.getValue() == State.EDGE_UP)) {
 992                 up = true;
 993             }
 994             break;
 995         default:
 996             return;
 997         }
 998
 999         if (up) {
1000             handleNodeConnectorStatusUp(nodeConnector);
1001         } else {
1002             handleNodeConnectorStatusDown(nodeConnector);
1003         }
1004     }
1005
1006     private void handleNodeConnectorStatusUp(NodeConnector nodeConnector) {
1007         if (topologyManager == null) {
1008             log.debug("topologyManager is not set yet");
1009             return;
1010         }
1011
1012         if (topologyManager.isInternal(nodeConnector)) {
1013             log.debug("{} is not a host facing link", nodeConnector);
1014             return;
1015         }
1016
1017         log.debug("{} is up", nodeConnector);
1018         updateRulesforHIFup(nodeConnector.getNode(), nodeConnector);
1019     }
1020
1021     private void handleNodeConnectorStatusDown(NodeConnector nodeConnector) {
1022         log.debug("{} is down", nodeConnector);
1023     }
1024
1025     void setClusterContainerService(IClusterContainerServices s) {
1026         log.debug("Cluster Service set");
1027         this.clusterContainerService = s;
1028     }
1029
1030     void unsetClusterContainerService(IClusterContainerServices s) {
1031         if (this.clusterContainerService == s) {
1032             log.debug("Cluster Service removed!");
1033             this.clusterContainerService = null;
1034         }
1035     }
1036
1037     /**
1038      * Function called by the dependency manager when all the required
1039      * dependencies are satisfied
1040      *
1041      */
1042     void init() {
1043         startUp();
1044     }
1045
1046     /**
1047      * Function called by the dependency manager when at least one
1048      * dependency become unsatisfied or when the component is shutting
1049      * down because for example bundle is being stopped.
1050      *
1051      */
1052     void destroy() {
1053     }
1054
1055     /**
1056      * Function called by dependency manager after "init ()" is called
1057      * and after the services provided by the class are registered in
1058      * the service registry
1059      *
1060      */
1061     void start() {
1062     }
1063
1064     /**
1065      * Function called by the dependency manager before the services
1066      * exported by the component are unregistered, this will be
1067      * followed by a "destroy ()" calls
1068      *
1069      */
1070     void stop() {
1071         pendingPacketsAgerTimer.cancel();
1072         pendingPacketDestinations.clear();
1073     }
1074
1075     public void setSwitchManager(ISwitchManager switchManager) {
1076         this.switchManager = switchManager;
1077     }
1078
1079     public void unsetSwitchManager(ISwitchManager switchManager) {
1080         if (this.switchManager == switchManager) {
1081             this.switchManager = null;
1082         }
1083     }
1084
1085     @Override
1086     public PacketResult receiveDataPacket(RawPacket inPkt) {
1087         if (inPkt == null) {
1088             return PacketResult.IGNORED;
1089         }
1090         log.trace("Received a frame of size: {}", inPkt.getPacketData().length);
1091         Packet formattedPak = this.dataPacketService.decodeDataPacket(inPkt);
1092         if (formattedPak instanceof Ethernet) {
1093             Object nextPak = formattedPak.getPayload();
1094             if (nextPak instanceof IPv4) {
1095                 log.trace("Handle punted IP packet: {}", formattedPak);
1096                 handlePuntedIPPacket((IPv4) nextPak, inPkt.getIncomingNodeConnector(), true);
1097             }
1098         }
1099         return PacketResult.IGNORED;
1100
1101     }
1102
1103     private void handlePuntedIPPacket(IPv4 pkt, NodeConnector incomingNodeConnector, boolean allowAddPending) {
1104         InetAddress dIP = NetUtils.getInetAddress(pkt.getDestinationAddress());
1105         if (dIP == null || hostTracker == null) {
1106             log.debug("Invalid param(s) in handlePuntedIPPacket.. DestIP: {}. hostTracker: {}", dIP, hostTracker);
1107             return;
1108         }
1109         HostNodeConnector destHost = hostTracker.hostFind(dIP);
1110         /*
1111          * In cases when incoming and outgoing connectors are in the same node, there is no need
1112          * to verify that there is a route. Because of that, we will only need routing.getRoute()
1113          * if we know that src and dst nodes are different.
1114          */
1115         if (destHost != null
1116                 && (incomingNodeConnector.getNode().equals(destHost.getnodeconnectorNode()) ||
1117                     routing == null ||
1118                     routing.getRoute(incomingNodeConnector.getNode(), destHost.getnodeconnectorNode()) != null)) {
1119
1120             log.trace("Host {} is at {}", dIP, destHost.getnodeConnector());
1121
1122             // If SimpleForwarding is aware of this host, it will try to install
1123             // a path. Forward packet until it's done.
1124             if (dataPacketService != null) {
1125
1126                 /*
1127                  * if we know where the host is and there's a path from where this
1128                  * packet was punted to where the host is, then attempt best effort delivery to the host
1129                  */
1130                 NodeConnector nc = destHost.getnodeConnector();
1131                 log.trace("Forwarding punted IP received at {} to {}", incomingNodeConnector, nc);
1132                 // re-encode the Ethernet packet (the parent of the IPv4 packet)
1133                 RawPacket rp = this.dataPacketService.encodeDataPacket(pkt.getParent());
1134                 rp.setOutgoingNodeConnector(nc);
1135                 this.dataPacketService.transmitDataPacket(rp);
1136             }
1137         } else if (allowAddPending) {
1138             // If we made it here, let's hang on to the punted packet, with hopes that its destination
1139             // will become available soon.
1140             addToPendingPackets(dIP, pkt, incomingNodeConnector);
1141         } else {
1142             log.warn("Dropping punted IP packet received at {} to Host {}", incomingNodeConnector, dIP);
1143         }
1144     }
1145 }