opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManager.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.forwardingrulesmanager.internal;
  10
  11 import java.io.FileNotFoundException;
  12 import java.io.IOException;
  13 import java.io.ObjectInputStream;
  14 import java.net.InetAddress;
  15 import java.net.UnknownHostException;
  16 import java.util.ArrayList;
  17 import java.util.Collections;
  18 import java.util.EnumSet;
  19 import java.util.HashSet;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Map.Entry;
  23 import java.util.Set;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.Callable;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentMap;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.ExecutorService;
  30 import java.util.concurrent.Executors;
  31 import java.util.concurrent.Future;
  32 import java.util.concurrent.LinkedBlockingQueue;
  33
  34 import org.eclipse.osgi.framework.console.CommandInterpreter;
  35 import org.eclipse.osgi.framework.console.CommandProvider;
  36 import org.opendaylight.controller.clustering.services.CacheConfigException;
  37 import org.opendaylight.controller.clustering.services.CacheExistException;
  38 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
  39 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
  40 import org.opendaylight.controller.clustering.services.IClusterServices;
  41 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
  42 import org.opendaylight.controller.sal.connection.ConnectionLocality;
  43 import org.opendaylight.controller.connectionmanager.IConnectionManager;
  44 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
  45 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
  46 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
  47 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  48 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
  49 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
  50 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
  51 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
  52 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
  53 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
  54 import org.opendaylight.controller.sal.action.Action;
  55 import org.opendaylight.controller.sal.action.ActionType;
  56 import org.opendaylight.controller.sal.action.Controller;
  57 import org.opendaylight.controller.sal.action.Flood;
  58 import org.opendaylight.controller.sal.action.Output;
  59 import org.opendaylight.controller.sal.action.PopVlan;
  60 import org.opendaylight.controller.sal.core.Config;
  61 import org.opendaylight.controller.sal.core.ContainerFlow;
  62 import org.opendaylight.controller.sal.core.IContainer;
  63 import org.opendaylight.controller.sal.core.IContainerListener;
  64 import org.opendaylight.controller.sal.core.Node;
  65 import org.opendaylight.controller.sal.core.NodeConnector;
  66 import org.opendaylight.controller.sal.core.Property;
  67 import org.opendaylight.controller.sal.core.UpdateType;
  68 import org.opendaylight.controller.sal.flowprogrammer.Flow;
  69 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
  70 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
  71 import org.opendaylight.controller.sal.match.Match;
  72 import org.opendaylight.controller.sal.match.MatchType;
  73 import org.opendaylight.controller.sal.utils.EtherTypes;
  74 import org.opendaylight.controller.sal.utils.GlobalConstants;
  75 import org.opendaylight.controller.sal.utils.HexEncode;
  76 import org.opendaylight.controller.sal.utils.IObjectReader;
  77 import org.opendaylight.controller.sal.utils.IPProtocols;
  78 import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
  79 import org.opendaylight.controller.sal.utils.NodeCreator;
  80 import org.opendaylight.controller.sal.utils.ObjectReader;
  81 import org.opendaylight.controller.sal.utils.ObjectWriter;
  82 import org.opendaylight.controller.sal.utils.Status;
  83 import org.opendaylight.controller.sal.utils.StatusCode;
  84 import org.opendaylight.controller.switchmanager.IInventoryListener;
  85 import org.opendaylight.controller.switchmanager.ISwitchManager;
  86 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
  87 import org.opendaylight.controller.switchmanager.Subnet;
  88 import org.osgi.framework.BundleContext;
  89 import org.osgi.framework.FrameworkUtil;
  90 import org.slf4j.Logger;
  91 import org.slf4j.LoggerFactory;
  92
  93 /**
  94  * Class that manages forwarding rule installation and removal per container of
  95  * the network. It also maintains the central repository of all the forwarding
  96  * rules installed on the network nodes.
  97  */
  98 public class ForwardingRulesManager implements
  99         IForwardingRulesManager,
 100         PortGroupChangeListener,
 101         IContainerListener,
 102         ISwitchManagerAware,
 103         IConfigurationContainerAware,
 104         IInventoryListener,
 105         IObjectReader,
 106         ICacheUpdateAware<Object,Object>,
 107         CommandProvider,
 108         IFlowProgrammerListener {
 109     private static final String NODEDOWN = "Node is Down";
 110     private static final String SUCCESS = StatusCode.SUCCESS.toString();
 111     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
 112     private static final String PORTREMOVED = "Port removed";
 113     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
 114     private String frmFileName;
 115     private String portGroupFileName;
 116     private ConcurrentMap<Integer, FlowConfig> staticFlows;
 117     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
 118     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
 119     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
 120     private ConcurrentMap<String, Object> TSPolicies;
 121     private boolean inContainerMode; // being used by global instance only
 122     private boolean stopping;
 123
 124     /*
 125      * Flow database. It's the software view of what was requested to install
 126      * and what is installed on the switch. It is indexed by the entry itself.
 127      * The entry's hashcode resumes the network node index, the flow's priority
 128      * and the flow's match. The value element is a class which contains the
 129      * flow entry pushed by the applications modules and the respective
 130      * container flow merged version. In absence of container flows, the two
 131      * flow entries are the same.
 132      */
 133     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
 134     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
 135     /*
 136      * Per node and per group indexing
 137      */
 138     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
 139     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
 140
 141     /*
 142      * Inactive flow list. This is for the global instance of FRM It will
 143      * contain all the flow entries which were installed on the global container
 144      * when the first container is created.
 145      */
 146     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
 147
 148     private IContainer container;
 149     private Set<IForwardingRulesManagerAware> frmAware =
 150         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
 151     private PortGroupProvider portGroupProvider;
 152     private IFlowProgrammerService programmer;
 153     private IClusterContainerServices clusterContainerService = null;
 154     private ISwitchManager switchManager;
 155     private Thread frmEventHandler;
 156     protected BlockingQueue<FRMEvent> pendingEvents;
 157
 158     // Distributes FRM programming in the cluster
 159     private IConnectionManager connectionManager;
 160
 161     /*
 162      * Name clustered caches used to support FRM entry distribution these are by
 163      * necessity non-transactional as long as need to be able to synchronize
 164      * states also while a transaction is in progress
 165      */
 166     static final String WORKORDERCACHE = "frm.workOrder";
 167     static final String WORKSTATUSCACHE = "frm.workStatus";
 168
 169     /*
 170      * Data structure responsible for distributing the FlowEntryInstall requests
 171      * in the cluster. The key value is entry that is being either Installed or
 172      * Updated or Delete. The value field is the same of the key value in case
 173      * of Installation or Deletion, it's the new entry in case of Modification,
 174      * this because the clustering caches don't allow null values.
 175      *
 176      * The logic behind this data structure is that the controller that initiate
 177      * the request will place the order here, someone will pick it and then will
 178      * remove from this data structure because is being served.
 179      *
 180      * TODO: We need to have a way to cleanup this data structure if entries are
 181      * not picked by anyone, which is always a case can happen especially on
 182      * Node disconnect cases.
 183      */
 184     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
 185
 186     /*
 187      * Data structure responsible for retrieving the results of the workOrder
 188      * submitted to the cluster.
 189      *
 190      * The logic behind this data structure is that the controller that has
 191      * executed the order will then place the result in workStatus signaling
 192      * that there was a success or a failure.
 193      *
 194      * TODO: The workStatus entries need to have a lifetime associated in case
 195      * of requestor controller leaving the cluster.
 196      */
 197     private ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
 198
 199     /*
 200      * Local Map used to hold the Future which a caller can use to monitor for
 201      * completion
 202      */
 203     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
 204             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
 205
 206     /**
 207      * @param e
 208      *            Entry being installed/updated/removed
 209      * @param u
 210      *            New entry will be placed after the update operation. Valid
 211      *            only for UpdateType.CHANGED, null for all the other cases
 212      * @param t
 213      *            Type of update
 214      * @return a Future object for monitoring the progress of the result, or
 215      *         null in case the processing should take place locally
 216      */
 217     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
 218             UpdateType t) {
 219         // A null entry it's an unexpected condition, anyway it's safe to keep
 220         // the handling local
 221         if (e == null) {
 222             return null;
 223         }
 224
 225         Node n = e.getNode();
 226         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
 227             // Create the work order and distribute it
 228             FlowEntryDistributionOrder fe =
 229                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
 230             // First create the monitor job
 231             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
 232             logsync.trace("Node {} not local so sending fe {}", n, fe);
 233             workMonitor.put(fe, ret);
 234             if (t.equals(UpdateType.CHANGED)) {
 235                 // Then distribute the work
 236                 workOrder.put(fe, u);
 237             } else {
 238                 // Then distribute the work
 239                 workOrder.put(fe, e);
 240             }
 241             logsync.trace("WorkOrder requested");
 242             // Now create an Handle to monitor the execution of the operation
 243             return ret;
 244         }
 245
 246         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
 247         return null;
 248     }
 249
 250     /**
 251      * Adds a flow entry onto the network node It runs various validity checks
 252      * and derive the final container flows merged entries that will be
 253      * attempted to be installed
 254      *
 255      * @param flowEntry
 256      *            the original flow entry application requested to add
 257      * @param async
 258      *            the flag indicating if this is a asynchronous request
 259      * @return the status of this request. In case of asynchronous call, it will
 260      *         contain the unique id assigned to this request
 261      */
 262     private Status addEntry(FlowEntry flowEntry, boolean async) {
 263
 264         // Sanity Check
 265         if (flowEntry == null || flowEntry.getNode() == null) {
 266             String msg = "Invalid FlowEntry";
 267             String logMsg = msg + ": {}";
 268             log.warn(logMsg, flowEntry);
 269             return new Status(StatusCode.NOTACCEPTABLE, msg);
 270         }
 271
 272         /*
 273          * Derive the container flow merged entries to install In presence of N
 274          * container flows, we may end up with N different entries to install...
 275          */
 276         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 277
 278         // Container Flow conflict Check
 279         if (toInstallList.isEmpty()) {
 280             String msg = "Flow Entry conflicts with all Container Flows";
 281             String logMsg = msg + ": {}";
 282             log.warn(logMsg, flowEntry);
 283             return new Status(StatusCode.CONFLICT, msg);
 284         }
 285
 286         // Derive the list of entries good to be installed
 287         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 288         for (FlowEntryInstall entry : toInstallList) {
 289             // Conflict Check: Verify new entry would not overwrite existing
 290             // ones
 291             if (this.installedSwView.containsKey(entry)) {
 292                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
 293                 log.trace("Aborting to install {}", entry);
 294                 continue;
 295             }
 296             toInstallSafe.add(entry);
 297         }
 298
 299         // Declare failure if all the container flow merged entries clash with
 300         // existing entries
 301         if (toInstallSafe.size() == 0) {
 302             String msg = "A flow with same match and priority exists on the target node";
 303             String logMsg = msg + ": {}";
 304             log.warn(logMsg, flowEntry);
 305             return new Status(StatusCode.CONFLICT, msg);
 306         }
 307
 308         // Try to install an entry at the time
 309         Status error = new Status(null, null);
 310         Status succeded = null;
 311         boolean oneSucceded = false;
 312         for (FlowEntryInstall installEntry : toInstallSafe) {
 313
 314             // Install and update database
 315             Status ret = addEntriesInternal(installEntry, async);
 316
 317             if (ret.isSuccess()) {
 318                 oneSucceded = true;
 319                 /*
 320                  * The first successful status response will be returned For the
 321                  * asynchronous call, we can discard the container flow
 322                  * complication for now and assume we will always deal with one
 323                  * flow only per request
 324                  */
 325                 succeded = ret;
 326             } else {
 327                 error = ret;
 328                 log.warn("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
 329             }
 330         }
 331
 332         return (oneSucceded) ? succeded : error;
 333     }
 334
 335     /**
 336      * Given a flow entry and the list of container flows, it returns the list
 337      * of container flow merged flow entries good to be installed on this
 338      * container. If the list of container flows is null or empty, the install
 339      * entry list will contain only one entry, the original flow entry. If the
 340      * flow entry is congruent with all the N container flows, then the output
 341      * install entry list will contain N entries. If the output list is empty,
 342      * it means the passed flow entry conflicts with all the container flows.
 343      *
 344      * @param cFlowList
 345      *            The list of container flows
 346      * @return the list of container flow merged entries good to be installed on
 347      *         this container
 348      */
 349     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
 350         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
 351
 352         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
 353             // No container flows => entry good to be installed unchanged
 354             toInstallList.add(new FlowEntryInstall(request.clone(), null));
 355         } else {
 356             // Create the list of entries to be installed. If the flow entry is
 357             // not congruent with any container flow, no install entries will be
 358             // created
 359             for (ContainerFlow cFlow : container.getContainerFlows()) {
 360                 if (cFlow.allowsFlow(request.getFlow())) {
 361                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
 362                 }
 363             }
 364         }
 365         return toInstallList;
 366     }
 367
 368     /**
 369      * Modify a flow entry with a new one It runs various validity check and
 370      * derive the final container flows merged flow entries to work with
 371      *
 372      * @param currentFlowEntry
 373      * @param newFlowEntry
 374      * @param async
 375      *            the flag indicating if this is a asynchronous request
 376      * @return the status of this request. In case of asynchronous call, it will
 377      *         contain the unique id assigned to this request
 378      */
 379     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
 380         Status retExt;
 381
 382         // Sanity checks
 383         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
 384                 || newFlowEntry.getNode() == null) {
 385             String msg = "Modify: Invalid FlowEntry";
 386             String logMsg = msg + ": {} or {}";
 387             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 388             return new Status(StatusCode.NOTACCEPTABLE, msg);
 389         }
 390         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
 391                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
 392             String msg = "Modify: Incompatible Flow Entries";
 393             String logMsg = msg + ": {} and {}";
 394             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 395             return new Status(StatusCode.NOTACCEPTABLE, msg);
 396         }
 397
 398         // Equality Check
 399         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
 400             String msg = "Modify skipped as flows are the same";
 401             String logMsg = msg + ": {} and {}";
 402             log.debug(logMsg, currentFlowEntry, newFlowEntry);
 403             return new Status(StatusCode.SUCCESS, msg);
 404         }
 405
 406         /*
 407          * Conflict Check: Verify the new entry would not conflict with an
 408          * existing one. This is a loose check on the previous original flow
 409          * entry requests. No check on the container flow merged flow entries
 410          * (if any) yet
 411          */
 412         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
 413         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
 414             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
 415             String logMsg = msg + ": {}";
 416             log.warn(logMsg, currentFlowEntry);
 417             return new Status(StatusCode.CONFLICT, msg);
 418         }
 419
 420         // Derive the installed and toInstall entries
 421         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
 422                 container.getContainerFlows());
 423         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
 424
 425         if (toInstallList.isEmpty()) {
 426             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
 427             String logMsg = msg + ": {}";
 428             log.warn(logMsg, newFlowEntry);
 429             log.warn(msg);
 430             return new Status(StatusCode.CONFLICT, msg);
 431         }
 432
 433         /*
 434          * If the two list sizes differ, it means the new flow entry does not
 435          * satisfy the same number of container flows the current entry does.
 436          * This is only possible when the new entry and current entry have
 437          * different match. In this scenario the modification would ultimately
 438          * be handled as a remove and add operations in the protocol plugin.
 439          *
 440          * Also, if any of the new flow entries would clash with an existing
 441          * one, we cannot proceed with the modify operation, because it would
 442          * fail for some entries and leave stale entries on the network node.
 443          * Modify path can be taken only if it can be performed completely, for
 444          * all entries.
 445          *
 446          * So, for the above two cases, to simplify, let's decouple the modify
 447          * in: 1) remove current entries 2) install new entries
 448          */
 449         Status succeeded = null;
 450         boolean decouple = false;
 451         if (installedList.size() != toInstallList.size()) {
 452             log.info("Modify: New flow entry does not satisfy the same "
 453                     + "number of container flows as the original entry does");
 454             decouple = true;
 455         }
 456         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 457         for (FlowEntryInstall installEntry : toInstallList) {
 458             /*
 459              * Conflict Check: Verify the new entry would not overwrite another
 460              * existing one
 461              */
 462             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
 463             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
 464                 log.info("Modify: new container flow merged flow entry clashes with existing flow");
 465                 decouple = true;
 466             } else {
 467                 toInstallSafe.add(installEntry);
 468             }
 469         }
 470
 471         if (decouple) {
 472             // Remove current entries
 473             for (FlowEntryInstall currEntry : installedList) {
 474                 this.removeEntryInternal(currEntry, async);
 475             }
 476             // Install new entries
 477             for (FlowEntryInstall newEntry : toInstallSafe) {
 478                 succeeded = this.addEntriesInternal(newEntry, async);
 479             }
 480         } else {
 481             /*
 482              * The two list have the same size and the entries to install do not
 483              * clash with any existing flow on the network node. We assume here
 484              * (and might be wrong) that the same container flows that were
 485              * satisfied by the current entries are the same that are satisfied
 486              * by the new entries. Let's take the risk for now.
 487              *
 488              * Note: modification has to be complete. If any entry modification
 489              * fails, we need to stop, restore the already modified entries, and
 490              * declare failure.
 491              */
 492             Status retModify = null;
 493             int i = 0;
 494             int size = toInstallList.size();
 495             while (i < size) {
 496                 // Modify and update database
 497                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
 498                 if (retModify.isSuccess()) {
 499                     i++;
 500                 } else {
 501                     break;
 502                 }
 503             }
 504             // Check if uncompleted modify
 505             if (i < size) {
 506                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
 507                 // Restore original entries
 508                 int j = 0;
 509                 while (j < i) {
 510                     log.info("Attempting to restore initial entries");
 511                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
 512                     if (retExt.isSuccess()) {
 513                         j++;
 514                     } else {
 515                         break;
 516                     }
 517                 }
 518                 // Fatal error, recovery failed
 519                 if (j < i) {
 520                     String msg = "Flow recovery failed ! Unrecoverable Error";
 521                     log.error(msg);
 522                     return new Status(StatusCode.INTERNALERROR, msg);
 523                 }
 524             }
 525             succeeded = retModify;
 526         }
 527         /*
 528          * The first successful status response will be returned. For the
 529          * asynchronous call, we can discard the container flow complication for
 530          * now and assume we will always deal with one flow only per request
 531          */
 532         return succeeded;
 533     }
 534
 535     /**
 536      * This is the function that modifies the final container flows merged
 537      * entries on the network node and update the database. It expects that all
 538      * the validity checks are passed
 539      *
 540      * @param currentEntries
 541      * @param newEntries
 542      * @param async
 543      *            the flag indicating if this is a asynchronous request
 544      * @return the status of this request. In case of asynchronous call, it will
 545      *         contain the unique id assigned to this request
 546      */
 547     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 548         FlowEntryDistributionOrderFutureTask futureStatus =
 549                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
 550         if (futureStatus != null) {
 551             Status retStatus = new Status(StatusCode.UNDEFINED);
 552             try {
 553                 retStatus = futureStatus.get();
 554                 if (retStatus.getCode()
 555                         .equals(StatusCode.TIMEOUT)) {
 556                     // A timeout happened, lets cleanup the workMonitor
 557                     workMonitor.remove(futureStatus.getOrder());
 558                 }
 559             } catch (InterruptedException e) {
 560                 log.error("", e);
 561             } catch (ExecutionException e) {
 562                 log.error("", e);
 563             }
 564             return retStatus;
 565         } else {
 566             // Modify the flow on the network node
 567             Status status = async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
 568                     .getFlow(), newEntries.getInstall()
 569                     .getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries.getInstall()
 570                     .getFlow(), newEntries.getInstall()
 571                     .getFlow());
 572
 573             if (!status.isSuccess()) {
 574                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
 575                         status.getDescription());
 576                 return status;
 577             }
 578
 579             log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
 580
 581             // Update DB
 582             newEntries.setRequestId(status.getRequestId());
 583             updateLocalDatabase(currentEntries, false);
 584             updateLocalDatabase(newEntries, true);
 585
 586             return status;
 587         }
 588     }
 589
 590     /**
 591      * Remove a flow entry. If the entry is not present in the software view
 592      * (entry or node not present), it return successfully
 593      *
 594      * @param flowEntry
 595      *            the flow entry to remove
 596      * @param async
 597      *            the flag indicating if this is a asynchronous request
 598      * @return the status of this request. In case of asynchronous call, it will
 599      *         contain the unique id assigned to this request
 600      */
 601     private Status removeEntry(FlowEntry flowEntry, boolean async) {
 602         Status error = new Status(null, null);
 603
 604         // Sanity Check
 605         if (flowEntry == null || flowEntry.getNode() == null) {
 606             String msg = "Invalid FlowEntry";
 607             String logMsg = msg + ": {}";
 608             log.warn(logMsg, flowEntry);
 609             return new Status(StatusCode.NOTACCEPTABLE, msg);
 610         }
 611
 612         // Derive the container flows merged installed entries
 613         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 614
 615         Status succeeded = null;
 616         boolean atLeastOneRemoved = false;
 617         for (FlowEntryInstall entry : installedList) {
 618             if (!installedSwView.containsKey(entry)) {
 619                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
 620                 log.debug(logMsg, flowEntry);
 621                 if (installedList.size() == 1) {
 622                     // If we had only one entry to remove, we are done
 623                     return new Status(StatusCode.SUCCESS);
 624                 } else {
 625                     continue;
 626                 }
 627             }
 628
 629             // Remove and update DB
 630             Status ret = removeEntryInternal(entry, async);
 631
 632             if (!ret.isSuccess()) {
 633                 error = ret;
 634                 log.warn("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
 635                 if (installedList.size() == 1) {
 636                     // If we had only one entry to remove, this is fatal failure
 637                     return error;
 638                 }
 639             } else {
 640                 succeeded = ret;
 641                 atLeastOneRemoved = true;
 642             }
 643         }
 644
 645         /*
 646          * No worries if full removal failed. Consistency checker will take care
 647          * of removing the stale entries later, or adjusting the software
 648          * database if not in sync with hardware
 649          */
 650         return (atLeastOneRemoved) ? succeeded : error;
 651     }
 652
 653     /**
 654      * This is the function that removes the final container flows merged entry
 655      * from the network node and update the database. It expects that all the
 656      * validity checks are passed
 657      *
 658      * @param entry
 659      *            the flow entry to remove
 660      * @param async
 661      *            the flag indicating if this is a asynchronous request
 662      * @return the status of this request. In case of asynchronous call, it will
 663      *         contain the unique id assigned to this request
 664      */
 665     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
 666         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
 667         if (futureStatus != null) {
 668             Status retStatus = new Status(StatusCode.UNDEFINED);
 669             try {
 670                 retStatus = futureStatus.get();
 671                 if (retStatus.getCode()
 672                         .equals(StatusCode.TIMEOUT)) {
 673                     // A timeout happened, lets cleanup the workMonitor
 674                     workMonitor.remove(futureStatus.getOrder());
 675                 }
 676             } catch (InterruptedException e) {
 677                 log.error("", e);
 678             } catch (ExecutionException e) {
 679                 log.error("", e);
 680             }
 681             return retStatus;
 682         } else {
 683             // Mark the entry to be deleted (for CC just in case we fail)
 684             entry.toBeDeleted();
 685
 686             // Remove from node
 687             Status status = async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall()
 688                     .getFlow()) : programmer.removeFlow(entry.getNode(), entry.getInstall()
 689                     .getFlow());
 690
 691             if (!status.isSuccess()) {
 692                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
 693                         status.getDescription());
 694                 return status;
 695             }
 696             log.trace("Removed  {}", entry.getInstall());
 697
 698             // Update DB
 699             updateLocalDatabase(entry, false);
 700
 701             return status;
 702         }
 703     }
 704
 705     /**
 706      * This is the function that installs the final container flow merged entry
 707      * on the network node and updates the database. It expects that all the
 708      * validity and conflict checks are passed. That means it does not check
 709      * whether this flow would conflict or overwrite an existing one.
 710      *
 711      * @param entry
 712      *            the flow entry to install
 713      * @param async
 714      *            the flag indicating if this is a asynchronous request
 715      * @return the status of this request. In case of asynchronous call, it will
 716      *         contain the unique id assigned to this request
 717      */
 718     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
 719         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
 720         if (futureStatus != null) {
 721             Status retStatus = new Status(StatusCode.UNDEFINED);
 722             try {
 723                 retStatus = futureStatus.get();
 724                 if (retStatus.getCode()
 725                         .equals(StatusCode.TIMEOUT)) {
 726                     // A timeout happened, lets cleanup the workMonitor
 727                     workMonitor.remove(futureStatus.getOrder());
 728                 }
 729             } catch (InterruptedException e) {
 730                 log.error("", e);
 731             } catch (ExecutionException e) {
 732                 log.error("", e);
 733             }
 734             return retStatus;
 735         } else {
 736             // Install the flow on the network node
 737             Status status = async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall()
 738                     .getFlow()) : programmer.addFlow(entry.getNode(), entry.getInstall()
 739                     .getFlow());
 740
 741             if (!status.isSuccess()) {
 742                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
 743                         status.getDescription());
 744                 return status;
 745             }
 746
 747             log.trace("Added    {}", entry.getInstall());
 748
 749             // Update DB
 750             entry.setRequestId(status.getRequestId());
 751             updateLocalDatabase(entry, true);
 752
 753             return status;
 754         }
 755     }
 756
 757     /**
 758      * Returns true if the flow conflicts with all the container's flows. This
 759      * means that if the function returns true, the passed flow entry is
 760      * congruent with at least one container flow, hence it is good to be
 761      * installed on this container.
 762      *
 763      * @param flowEntry
 764      * @return true if flow conflicts with all the container flows, false
 765      *         otherwise
 766      */
 767     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
 768         List<ContainerFlow> cFlowList = container.getContainerFlows();
 769
 770         // Validity check and avoid unnecessary computation
 771         // Also takes care of default container where no container flows are
 772         // present
 773         if (cFlowList == null || cFlowList.isEmpty()) {
 774             return false;
 775         }
 776
 777         for (ContainerFlow cFlow : cFlowList) {
 778             if (cFlow.allowsFlow(flowEntry.getFlow())) {
 779                 // Entry is allowed by at least one container flow: good to go
 780                 return false;
 781             }
 782         }
 783         return true;
 784     }
 785
 786     private void updateLocalDatabase(FlowEntryInstall entry, boolean add) {
 787         // Update the software view
 788         updateSwViewes(entry, add);
 789
 790         // Update node indexed flow database
 791         updateNodeFlowsDB(entry, add);
 792
 793         // Update group indexed flow database
 794         updateGroupFlowsDB(entry, add);
 795     }
 796
 797     /*
 798      * Update the node mapped flows database
 799      */
 800     private void updateSwViewes(FlowEntryInstall flowEntries, boolean add) {
 801         if (add) {
 802             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
 803             installedSwView.put(flowEntries, flowEntries);
 804         } else {
 805             originalSwView.remove(flowEntries.getOriginal());
 806             installedSwView.remove(flowEntries);
 807         }
 808     }
 809
 810     /*
 811      * Update the node mapped flows database
 812      */
 813     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 814         Node node = flowEntries.getNode();
 815
 816         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
 817         if (nodeIndeces == null) {
 818             if (!add) {
 819                 return;
 820             } else {
 821                 nodeIndeces = new ArrayList<FlowEntryInstall>();
 822             }
 823         }
 824
 825         if (add) {
 826             nodeIndeces.add(flowEntries);
 827         } else {
 828             nodeIndeces.remove(flowEntries);
 829         }
 830
 831         // Update cache across cluster
 832         if (nodeIndeces.isEmpty()) {
 833             this.nodeFlows.remove(node);
 834         } else {
 835             this.nodeFlows.put(node, nodeIndeces);
 836         }
 837     }
 838
 839     /*
 840      * Update the group name mapped flows database
 841      */
 842     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 843         String groupName = flowEntries.getGroupName();
 844
 845         // Flow may not be part of a group
 846         if (groupName == null) {
 847             return;
 848         }
 849
 850         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
 851         if (indices == null) {
 852             if (!add) {
 853                 return;
 854             } else {
 855                 indices = new ArrayList<FlowEntryInstall>();
 856             }
 857         }
 858
 859         if (add) {
 860             indices.add(flowEntries);
 861         } else {
 862             indices.remove(flowEntries);
 863         }
 864
 865         // Update cache across cluster
 866         if (indices.isEmpty()) {
 867             this.groupFlows.remove(groupName);
 868         } else {
 869             this.groupFlows.put(groupName, indices);
 870         }
 871     }
 872
 873     /**
 874      * Remove a flow entry that has been added previously First checks if the
 875      * entry is effectively present in the local database
 876      */
 877     @SuppressWarnings("unused")
 878     private Status removeEntry(Node node, String flowName) {
 879         FlowEntryInstall target = null;
 880
 881         // Find in database
 882         for (FlowEntryInstall entry : installedSwView.values()) {
 883             if (entry.equalsByNodeAndName(node, flowName)) {
 884                 target = entry;
 885                 break;
 886             }
 887         }
 888
 889         // If it is not there, stop any further processing
 890         if (target == null) {
 891             return new Status(StatusCode.SUCCESS, "Entry is not present");
 892         }
 893
 894         // Remove from node
 895         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
 896
 897         // Update DB
 898         if (status.isSuccess()) {
 899             updateLocalDatabase(target, false);
 900         } else {
 901             // log the error
 902             log.warn("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
 903                     status.getDescription());
 904         }
 905
 906         return status;
 907     }
 908
 909     @Override
 910     public Status installFlowEntry(FlowEntry flowEntry) {
 911         Status status;
 912         if (isContainerModeAllowed(flowEntry)) {
 913             status = addEntry(flowEntry, false);
 914         } else {
 915             String msg = "Controller in container mode: Install Refused";
 916             String logMsg = msg + ": {}";
 917             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 918             log.warn(logMsg, flowEntry);
 919         }
 920         return status;
 921     }
 922
 923     @Override
 924     public Status installFlowEntryAsync(FlowEntry flowEntry) {
 925         Status status;
 926         if (isContainerModeAllowed(flowEntry)) {
 927             status = addEntry(flowEntry, true);
 928         } else {
 929             String msg = "Controller in container mode: Install Refused";
 930             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 931             log.warn(msg);
 932         }
 933         return status;
 934     }
 935
 936     @Override
 937     public Status uninstallFlowEntry(FlowEntry flowEntry) {
 938         Status status;
 939         if (isContainerModeAllowed(flowEntry)) {
 940             status = removeEntry(flowEntry, false);
 941         } else {
 942             String msg = "Controller in container mode: Uninstall Refused";
 943             String logMsg = msg + ": {}";
 944             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 945             log.warn(logMsg, flowEntry);
 946         }
 947         return status;
 948     }
 949
 950     @Override
 951     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
 952         Status status;
 953         if (isContainerModeAllowed(flowEntry)) {
 954             status = removeEntry(flowEntry, true);
 955         } else {
 956             String msg = "Controller in container mode: Uninstall Refused";
 957             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 958             log.warn(msg);
 959         }
 960         return status;
 961     }
 962
 963     @Override
 964     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
 965         Status status = null;
 966         if (isContainerModeAllowed(currentFlowEntry)) {
 967             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
 968         } else {
 969             String msg = "Controller in container mode: Modify Refused";
 970             String logMsg = msg + ": {}";
 971             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 972             log.warn(logMsg, newFlowEntry);
 973         }
 974         return status;
 975     }
 976
 977     @Override
 978     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
 979         Status status = null;
 980         if (isContainerModeAllowed(currentFlowEntry)) {
 981             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
 982         } else {
 983             String msg = "Controller in container mode: Modify Refused";
 984             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 985             log.warn(msg);
 986         }
 987         return status;
 988     }
 989
 990     /**
 991      * Returns whether the specified flow entry is allowed to be
 992      * installed/removed/modified based on the current container mode status.
 993      * This call always returns true in the container instance of forwarding
 994      * rules manager. It is meant for the global instance only (default
 995      * container) of forwarding rules manager. Idea is that for assuring
 996      * container isolation of traffic, flow installation in default container is
 997      * blocked when in container mode (containers are present). The only flows
 998      * that are allowed in container mode in the default container are the
 999      * proactive flows, the ones automatically installed on the network node
1000      * which forwarding mode has been configured to "proactive". These flows are
1001      * needed by controller to discover the nodes topology and to discover the
1002      * attached hosts for some SDN switches.
1003      *
1004      * @param flowEntry
1005      *            The flow entry to be installed/removed/modified
1006      * @return true if not in container mode or if flowEntry is internally
1007      *         generated
1008      */
1009     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1010         return (!inContainerMode) ? true : flowEntry.isInternal();
1011     }
1012
1013     @Override
1014     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1015         /*
1016          * Run a check on the original entries to decide whether to go with a
1017          * add or modify method. A loose check means only check against the
1018          * original flow entry requests and not against the installed flow
1019          * entries which are the result of the original entry merged with the
1020          * container flow(s) (if any). The modifyFlowEntry method in presence of
1021          * conflicts with the Container flows (if any) would revert back to a
1022          * delete + add pattern
1023          */
1024         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1025
1026         if (currentFlowEntry != null) {
1027             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1028         } else {
1029             return installFlowEntry(newFlowEntry);
1030         }
1031     }
1032
1033     @Override
1034     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1035         /*
1036          * Run a check on the original entries to decide whether to go with a
1037          * add or modify method. A loose check means only check against the
1038          * original flow entry requests and not against the installed flow
1039          * entries which are the result of the original entry merged with the
1040          * container flow(s) (if any). The modifyFlowEntry method in presence of
1041          * conflicts with the Container flows (if any) would revert back to a
1042          * delete + add pattern
1043          */
1044         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1045
1046         if (currentFlowEntry != null) {
1047             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1048         } else {
1049             return installFlowEntryAsync(newFlowEntry);
1050         }
1051     }
1052
1053     @Override
1054     public Status uninstallFlowEntryGroup(String groupName) {
1055         if (groupName == null || groupName.isEmpty()) {
1056             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1057         }
1058         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1059             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1060         }
1061         if (inContainerMode) {
1062             String msg = "Controller in container mode: Group Uninstall Refused";
1063             String logMsg = msg + ": {}";
1064             log.warn(logMsg, groupName);
1065             return new Status(StatusCode.NOTACCEPTABLE, msg);
1066         }
1067         int toBeRemoved = 0;
1068         String error = "";
1069         if (groupFlows.containsKey(groupName)) {
1070             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1071             toBeRemoved = list.size();
1072             for (FlowEntryInstall entry : list) {
1073                 Status status = this.removeEntry(entry.getOriginal(), false);
1074                 if (status.isSuccess()) {
1075                     toBeRemoved -= 1;
1076                 } else {
1077                     error = status.getDescription();
1078                 }
1079             }
1080         }
1081         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1082                 "Not all the flows were removed: " + error);
1083     }
1084
1085     @Override
1086     public Status uninstallFlowEntryGroupAsync(String groupName) {
1087         if (groupName == null || groupName.isEmpty()) {
1088             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1089         }
1090         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1091             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1092         }
1093         if (inContainerMode) {
1094             String msg = "Controller in container mode: Group Uninstall Refused";
1095             String logMsg = msg + ": {}";
1096             log.warn(logMsg, groupName);
1097             return new Status(StatusCode.NOTACCEPTABLE, msg);
1098         }
1099         if (groupFlows.containsKey(groupName)) {
1100             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1101             for (FlowEntryInstall entry : list) {
1102                 this.removeEntry(entry.getOriginal(), true);
1103             }
1104         }
1105         return new Status(StatusCode.SUCCESS);
1106     }
1107
1108     @Override
1109     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1110         return entryConflictsWithContainerFlows(flowEntry);
1111     }
1112
1113     /**
1114      * Updates all installed flows because the container flow got updated This
1115      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1116      * Reinstall of all flows This is needed because a new container flows
1117      * merged flow may conflict with an existing old container flows merged flow
1118      * on the network node
1119      */
1120     private void updateFlowsContainerFlow() {
1121         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1122         // First remove all installed entries
1123         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1124             FlowEntryInstall current = entry.getValue();
1125             // Store the original entry
1126             toReInstall.add(current.getOriginal());
1127             // Remove the old couples. No validity checks to be run, use the
1128             // internal remove
1129             this.removeEntryInternal(current, false);
1130         }
1131         // Then reinstall the original entries
1132         for (FlowEntry entry : toReInstall) {
1133             // Reinstall the original flow entries, via the regular path: new
1134             // cFlow merge + validations
1135             this.installFlowEntry(entry);
1136         }
1137     }
1138
1139     private void nonClusterObjectCreate() {
1140         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1141         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1142         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
1143         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
1144         TSPolicies = new ConcurrentHashMap<String, Object>();
1145         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1146         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1147         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1148         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1149         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1150     }
1151
1152     private void registerWithOSGIConsole() {
1153         BundleContext bundleContext = FrameworkUtil.getBundle(this.getClass()).getBundleContext();
1154         bundleContext.registerService(CommandProvider.class.getName(), this, null);
1155     }
1156
1157     @Override
1158     public void setTSPolicyData(String policyname, Object o, boolean add) {
1159
1160         if (add) {
1161             /* Check if this policy already exists */
1162             if (!(TSPolicies.containsKey(policyname))) {
1163                 TSPolicies.put(policyname, o);
1164             }
1165         } else {
1166             TSPolicies.remove(policyname);
1167         }
1168         if (frmAware != null) {
1169             synchronized (frmAware) {
1170                 for (IForwardingRulesManagerAware frma : frmAware) {
1171                     try {
1172                         frma.policyUpdate(policyname, add);
1173                     } catch (Exception e) {
1174                         log.warn("Exception on callback", e);
1175                     }
1176                 }
1177             }
1178         }
1179     }
1180
1181     @Override
1182     public Map<String, Object> getTSPolicyData() {
1183         return TSPolicies;
1184     }
1185
1186     @Override
1187     public Object getTSPolicyData(String policyName) {
1188         if (TSPolicies.containsKey(policyName)) {
1189             return TSPolicies.get(policyName);
1190         } else {
1191             return null;
1192         }
1193     }
1194
1195     @Override
1196     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1197         List<FlowEntry> list = new ArrayList<FlowEntry>();
1198         if (policyName != null && !policyName.trim().isEmpty()) {
1199             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1200                 if (policyName.equals(entry.getKey().getGroupName())) {
1201                     list.add(entry.getKey().clone());
1202                 }
1203             }
1204         }
1205         return list;
1206     }
1207
1208     @Override
1209     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1210         List<FlowEntry> list = new ArrayList<FlowEntry>();
1211         if (policyName != null && !policyName.trim().isEmpty()) {
1212             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1213                 if (policyName.equals(entry.getKey().getGroupName())) {
1214                     list.add(entry.getKey().getInstall().clone());
1215                 }
1216             }
1217         }
1218         return list;
1219     }
1220
1221     @Override
1222     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1223
1224         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1225             if (flow.getFlowName().equals(flowName)) {
1226                 FlowEntry currentFlowEntry = flow.getOriginal();
1227                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1228                 for (NodeConnector dstPort : portList) {
1229                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1230                 }
1231                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1232                 if (error.isSuccess()) {
1233                     log.info("Ports {} added to FlowEntry {}", portList, flowName);
1234                 } else {
1235                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1236                             currentFlowEntry.toString(), error.getDescription());
1237                 }
1238                 return;
1239             }
1240         }
1241         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1242     }
1243
1244     @Override
1245     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1246         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1247             FlowEntryInstall flow = this.installedSwView.get(index);
1248             if (flow.getFlowName().equals(flowName)) {
1249                 FlowEntry currentFlowEntry = flow.getOriginal();
1250                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1251                 for (NodeConnector dstPort : portList) {
1252                     Action action = new Output(dstPort);
1253                     newFlowEntry.getFlow().removeAction(action);
1254                 }
1255                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1256                 if (status.isSuccess()) {
1257                     log.info("Ports {} removed from FlowEntry {}", portList, flowName);
1258                 } else {
1259                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1260                             currentFlowEntry.toString(), status.getDescription());
1261                 }
1262                 return;
1263             }
1264         }
1265         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1266     }
1267
1268     /*
1269      * This function assumes the target flow has only one output port
1270      */
1271     @Override
1272     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1273         FlowEntry currentFlowEntry = null;
1274         FlowEntry newFlowEntry = null;
1275
1276         // Find the flow
1277         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1278             FlowEntryInstall flow = this.installedSwView.get(index);
1279             if (flow.getFlowName().equals(flowName)) {
1280                 currentFlowEntry = flow.getOriginal();
1281                 break;
1282             }
1283         }
1284         if (currentFlowEntry == null) {
1285             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1286             return;
1287         }
1288
1289         // Create a flow copy with the new output port
1290         newFlowEntry = currentFlowEntry.clone();
1291         Action target = null;
1292         for (Action action : newFlowEntry.getFlow().getActions()) {
1293             if (action.getType() == ActionType.OUTPUT) {
1294                 target = action;
1295                 break;
1296             }
1297         }
1298         newFlowEntry.getFlow().removeAction(target);
1299         newFlowEntry.getFlow().addAction(new Output(outPort));
1300
1301         // Modify on network node
1302         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1303
1304         if (status.isSuccess()) {
1305             log.info("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1306         } else {
1307             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1308                     status.getDescription());
1309         }
1310         return;
1311     }
1312
1313     @Override
1314     public NodeConnector getOutputPort(Node node, String flowName) {
1315         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1316             FlowEntryInstall flow = this.installedSwView.get(index);
1317             if (flow.getFlowName().equals(flowName)) {
1318                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1319                     if (action.getType() == ActionType.OUTPUT) {
1320                         return ((Output) action).getPort();
1321                     }
1322                 }
1323             }
1324         }
1325         return null;
1326     }
1327
1328     private void cacheStartup() {
1329         allocateCaches();
1330         retrieveCaches();
1331     }
1332
1333     private void allocateCaches() {
1334         if (this.clusterContainerService == null) {
1335             log.warn("Un-initialized clusterContainerService, can't create cache");
1336             return;
1337         }
1338
1339         log.debug("Allocating caches for Container {}", container.getName());
1340
1341         try {
1342             clusterContainerService.createCache("frm.originalSwView",
1343                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1344
1345             clusterContainerService.createCache("frm.installedSwView",
1346                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1347
1348             clusterContainerService.createCache("frm.inactiveFlows",
1349                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1350
1351             clusterContainerService.createCache("frm.nodeFlows",
1352                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1353
1354             clusterContainerService.createCache("frm.groupFlows",
1355                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1356
1357             clusterContainerService.createCache("frm.staticFlows",
1358                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1359
1360             clusterContainerService.createCache("frm.flowsSaveEvent",
1361                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1362
1363             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1364                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1365
1366             clusterContainerService.createCache("frm.portGroupConfigs",
1367                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1368
1369             clusterContainerService.createCache("frm.portGroupData",
1370                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1371
1372             clusterContainerService.createCache("frm.TSPolicies",
1373                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1374
1375             clusterContainerService.createCache(WORKSTATUSCACHE,
1376  EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
1377
1378             clusterContainerService.createCache(WORKORDERCACHE,
1379  EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
1380
1381         } catch (CacheConfigException cce) {
1382             log.error("CacheConfigException");
1383         } catch (CacheExistException cce) {
1384             log.error("CacheExistException");
1385         }
1386     }
1387
1388     @SuppressWarnings({ "unchecked" })
1389     private void retrieveCaches() {
1390         ConcurrentMap<?, ?> map;
1391
1392         if (this.clusterContainerService == null) {
1393             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1394             nonClusterObjectCreate();
1395             return;
1396         }
1397
1398         log.debug("Retrieving Caches for Container {}", container.getName());
1399
1400         map = clusterContainerService.getCache("frm.originalSwView");
1401         if (map != null) {
1402             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1403         } else {
1404             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1405         }
1406
1407         map = clusterContainerService.getCache("frm.installedSwView");
1408         if (map != null) {
1409             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1410         } else {
1411             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1412         }
1413
1414         map = clusterContainerService.getCache("frm.inactiveFlows");
1415         if (map != null) {
1416             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1417         } else {
1418             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1419         }
1420
1421         map = clusterContainerService.getCache("frm.nodeFlows");
1422         if (map != null) {
1423             nodeFlows = (ConcurrentMap<Node, List<FlowEntryInstall>>) map;
1424         } else {
1425             log.error("Retrieval of cache failed for Container {}", container.getName());
1426         }
1427
1428         map = clusterContainerService.getCache("frm.groupFlows");
1429         if (map != null) {
1430             groupFlows = (ConcurrentMap<String, List<FlowEntryInstall>>) map;
1431         } else {
1432             log.error("Retrieval of frm.groupFlows cache failed for Container {}", container.getName());
1433         }
1434
1435         map = clusterContainerService.getCache("frm.staticFlows");
1436         if (map != null) {
1437             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1438         } else {
1439             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1440         }
1441
1442         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1443         if (map != null) {
1444             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1445         } else {
1446             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1447         }
1448
1449         map = clusterContainerService.getCache("frm.portGroupConfigs");
1450         if (map != null) {
1451             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1452         } else {
1453             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1454         }
1455
1456         map = clusterContainerService.getCache("frm.portGroupData");
1457         if (map != null) {
1458             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1459         } else {
1460             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1461         }
1462
1463         map = clusterContainerService.getCache("frm.TSPolicies");
1464         if (map != null) {
1465             TSPolicies = (ConcurrentMap<String, Object>) map;
1466         } else {
1467             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1468         }
1469
1470         map = clusterContainerService.getCache(WORKORDERCACHE);
1471         if (map != null) {
1472             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1473         } else {
1474             log.error("Retrieval of " + WORKORDERCACHE + " cache failed for Container {}", container.getName());
1475         }
1476
1477         map = clusterContainerService.getCache(WORKSTATUSCACHE);
1478         if (map != null) {
1479             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1480         } else {
1481             log.error("Retrieval of " + WORKSTATUSCACHE + " cache failed for Container {}", container.getName());
1482         }
1483     }
1484
1485     private boolean flowConfigExists(FlowConfig config) {
1486         // Flow name has to be unique on per node id basis
1487         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1488             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1489                 return true;
1490             }
1491         }
1492         return false;
1493     }
1494
1495     @Override
1496     public Status addStaticFlow(FlowConfig config) {
1497         // Configuration object validation
1498         Status status = config.validate(container);
1499         if (!status.isSuccess()) {
1500             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1501             String error = "Invalid Configuration (" + status.getDescription() + ")";
1502             config.setStatus(error);
1503             return new Status(StatusCode.BADREQUEST, error);
1504         }
1505         return addStaticFlowInternal(config, false);
1506     }
1507
1508     /**
1509      * Private method to add a static flow configuration which does not run any
1510      * validation on the passed FlowConfig object. If restore is set to true,
1511      * configuration is stored in configuration database regardless the
1512      * installation on the network node was successful. This is useful at boot
1513      * when static flows are present in startup configuration and are read
1514      * before the switches connects.
1515      *
1516      * @param config
1517      *            The static flow configuration
1518      * @param restore
1519      *            if true, the configuration is stored regardless the
1520      *            installation on the network node was successful
1521      * @return The status of this request
1522      */
1523     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1524         boolean multipleFlowPush = false;
1525         String error;
1526         Status status;
1527         config.setStatus(SUCCESS);
1528
1529         // Presence check
1530         if (flowConfigExists(config)) {
1531             error = "Entry with this name on specified switch already exists";
1532             log.warn("Entry with this name on specified switch already exists: {}", config);
1533             config.setStatus(error);
1534             return new Status(StatusCode.CONFLICT, error);
1535         }
1536
1537         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1538             for (String portGroupName : portGroupConfigs.keySet()) {
1539                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1540                     multipleFlowPush = true;
1541                     break;
1542                 }
1543             }
1544             if (!multipleFlowPush) {
1545                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1546                 error = "Invalid Configuration (Invalid PortGroup Name)";
1547                 config.setStatus(error);
1548                 return new Status(StatusCode.BADREQUEST, error);
1549             }
1550         }
1551
1552         /*
1553          * If requested program the entry in hardware first before updating the
1554          * StaticFlow DB
1555          */
1556         if (!multipleFlowPush) {
1557             // Program hw
1558             if (config.installInHw()) {
1559                 FlowEntry entry = config.getFlowEntry();
1560                 status = this.installFlowEntry(entry);
1561                 if (!status.isSuccess()) {
1562                     config.setStatus(status.getDescription());
1563                     if (!restore) {
1564                         return status;
1565                     }
1566                 }
1567             }
1568         }
1569
1570         /*
1571          * When the control reaches this point, either of the following
1572          * conditions is true 1. This is a single entry configuration (non
1573          * PortGroup) and the hardware installation is successful 2. This is a
1574          * multiple entry configuration (PortGroup) and hardware installation is
1575          * NOT done directly on this event. 3. The User prefers to retain the
1576          * configuration in Controller and skip hardware installation.
1577          *
1578          * Hence it is safe to update the StaticFlow DB at this point.
1579          *
1580          * Note : For the case of PortGrouping, it is essential to have this DB
1581          * populated before the PortGroupListeners can query for the DB
1582          * triggered using portGroupChanged event...
1583          */
1584         Integer ordinal = staticFlowsOrdinal.get(0);
1585         staticFlowsOrdinal.put(0, ++ordinal);
1586         staticFlows.put(ordinal, config);
1587
1588         if (multipleFlowPush) {
1589             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1590             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1591             if (existingData != null) {
1592                 portGroupChanged(pgconfig, existingData, true);
1593             }
1594         }
1595         return new Status(StatusCode.SUCCESS);
1596     }
1597
1598     private void addStaticFlowsToSwitch(Node node) {
1599         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1600             FlowConfig config = entry.getValue();
1601             if (config.isPortGroupEnabled()) {
1602                 continue;
1603             }
1604             if (config.getNode().equals(node)) {
1605                 if (config.installInHw() && !config.getStatus().equals(SUCCESS)) {
1606                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1607                     config.setStatus(status.getDescription());
1608                 }
1609             }
1610         }
1611         // Update cluster cache
1612         refreshClusterStaticFlowsStatus(node);
1613     }
1614
1615     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1616         log.trace("Updating Static Flow configs on node down: {}", node);
1617
1618         List<Integer> toRemove = new ArrayList<Integer>();
1619         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1620
1621             FlowConfig config = entry.getValue();
1622
1623             if (config.isPortGroupEnabled()) {
1624                 continue;
1625             }
1626
1627             if (config.installInHw() && config.getNode().equals(node)) {
1628                 if (config.isInternalFlow()) {
1629                     // Take note of this controller generated static flow
1630                     toRemove.add(entry.getKey());
1631                 } else {
1632                     config.setStatus(NODEDOWN);
1633                 }
1634             }
1635         }
1636         // Remove controller generated static flows for this node
1637         for (Integer index : toRemove) {
1638             staticFlows.remove(index);
1639         }
1640         // Update cluster cache
1641         refreshClusterStaticFlowsStatus(node);
1642
1643     }
1644
1645     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1646         log.trace("Updating Static Flow configs on container mode change: {}", update);
1647
1648         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1649             FlowConfig config = entry.getValue();
1650             if (config.isPortGroupEnabled()) {
1651                 continue;
1652             }
1653             if (config.installInHw() && !config.isInternalFlow()) {
1654                 switch (update) {
1655                 case ADDED:
1656                     config.setStatus("Removed from node because in container mode");
1657                     break;
1658                 case REMOVED:
1659                     config.setStatus(SUCCESS);
1660                     break;
1661                 default:
1662                 }
1663             }
1664         }
1665         // Update cluster cache
1666         refreshClusterStaticFlowsStatus(null);
1667     }
1668
1669     @Override
1670     public Status removeStaticFlow(FlowConfig config) {
1671         /*
1672          * No config.isInternal() check as NB does not take this path and GUI
1673          * cannot issue a delete on an internal generated flow. We need this
1674          * path to be accessible when switch mode is changed from proactive to
1675          * reactive, so that we can remove the internal generated LLDP and ARP
1676          * punt flows
1677          */
1678
1679         // Look for the target configuration entry
1680         Integer key = 0;
1681         FlowConfig target = null;
1682         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1683             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1684                 key = entry.getKey();
1685                 target = entry.getValue();
1686                 break;
1687             }
1688         }
1689         if (target == null) {
1690             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1691         }
1692
1693         // Program the network node
1694         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1695
1696         // Update configuration database if programming was successful
1697         if (status.isSuccess()) {
1698             staticFlows.remove(key);
1699         }
1700
1701         return status;
1702     }
1703
1704     @Override
1705     public Status removeStaticFlow(String name, Node node) {
1706         // Look for the target configuration entry
1707         Integer key = 0;
1708         FlowConfig target = null;
1709         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1710             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1711                 key = mapEntry.getKey();
1712                 target = mapEntry.getValue();
1713                 break;
1714             }
1715         }
1716         if (target == null) {
1717             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1718         }
1719
1720         // Validity check for api3 entry point
1721         if (target.isInternalFlow()) {
1722             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1723             String logMsg = msg + ": {}";
1724             log.warn(logMsg, name);
1725             return new Status(StatusCode.NOTACCEPTABLE, msg);
1726         }
1727
1728         if (target.isPortGroupEnabled()) {
1729             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1730             String logMsg = msg + ": {}";
1731             log.warn(logMsg, name);
1732             return new Status(StatusCode.NOTACCEPTABLE, msg);
1733         }
1734
1735         // Program the network node
1736         Status status = this.removeEntry(target.getFlowEntry(), false);
1737
1738         // Update configuration database if programming was successful
1739         if (status.isSuccess()) {
1740             staticFlows.remove(key);
1741         }
1742
1743         return status;
1744     }
1745
1746     @Override
1747     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1748         // Validity check for api3 entry point
1749         if (newFlowConfig.isInternalFlow()) {
1750             String msg = "Invalid operation: Controller generated flow cannot be modified";
1751             String logMsg = msg + ": {}";
1752             log.warn(logMsg, newFlowConfig);
1753             return new Status(StatusCode.NOTACCEPTABLE, msg);
1754         }
1755
1756         // Validity Check
1757         Status status = newFlowConfig.validate(container);
1758         if (!status.isSuccess()) {
1759             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1760             newFlowConfig.setStatus(msg);
1761             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1762             return new Status(StatusCode.BADREQUEST, msg);
1763         }
1764
1765         FlowConfig oldFlowConfig = null;
1766         Integer index = null;
1767         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1768             FlowConfig entry = mapEntry.getValue();
1769             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1770                 oldFlowConfig = entry;
1771                 index = mapEntry.getKey();
1772                 break;
1773             }
1774         }
1775
1776         if (oldFlowConfig == null) {
1777             String msg = "Attempt to modify a non existing static flow";
1778             String logMsg = msg + ": {}";
1779             log.warn(logMsg, newFlowConfig);
1780             return new Status(StatusCode.NOTFOUND, msg);
1781         }
1782
1783         // Do not attempt to reinstall the flow, warn user
1784         if (newFlowConfig.equals(oldFlowConfig)) {
1785             String msg = "No modification detected";
1786             log.info("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1787             return new Status(StatusCode.SUCCESS, msg);
1788         }
1789
1790         // If flow is installed, program the network node
1791         status = new Status(StatusCode.SUCCESS, "Saved in config");
1792         if (oldFlowConfig.installInHw()) {
1793             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1794         }
1795
1796         // Update configuration database if programming was successful
1797         if (status.isSuccess()) {
1798             newFlowConfig.setStatus(status.getDescription());
1799             staticFlows.put(index, newFlowConfig);
1800         }
1801
1802         return status;
1803     }
1804
1805     @Override
1806     public Status toggleStaticFlowStatus(String name, Node node) {
1807         return toggleStaticFlowStatus(getStaticFlow(name, node));
1808     }
1809
1810     @Override
1811     public Status toggleStaticFlowStatus(FlowConfig config) {
1812         if (config == null) {
1813             String msg = "Invalid request: null flow config";
1814             log.warn(msg);
1815             return new Status(StatusCode.BADREQUEST, msg);
1816         }
1817         // Validity check for api3 entry point
1818         if (config.isInternalFlow()) {
1819             String msg = "Invalid operation: Controller generated flow cannot be modified";
1820             String logMsg = msg + ": {}";
1821             log.warn(logMsg, config);
1822             return new Status(StatusCode.NOTACCEPTABLE, msg);
1823         }
1824
1825         // Find the config entry
1826         Integer key = 0;
1827         FlowConfig target = null;
1828         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1829             FlowConfig conf = entry.getValue();
1830             if (conf.isByNameAndNodeIdEqual(config)) {
1831                 key = entry.getKey();
1832                 target = conf;
1833                 break;
1834             }
1835         }
1836         if (target != null) {
1837             Status status = target.validate(container);
1838             if (!status.isSuccess()) {
1839                 log.warn(status.getDescription());
1840                 return status;
1841             }
1842             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1843                                     .installFlowEntry(target.getFlowEntry());
1844             if (status.isSuccess()) {
1845                 // Update Configuration database
1846                 target.setStatus(SUCCESS);
1847                 target.toggleInstallation();
1848                 staticFlows.put(key, target);
1849             }
1850             return status;
1851         }
1852
1853         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1854     }
1855
1856     /**
1857      * Reinsert all static flows entries in the cache to force cache updates in
1858      * the cluster. This is useful when only some parameters were changed in the
1859      * entries, like the status.
1860      *
1861      * @param node
1862      *            The node for which the static flow configurations have to be
1863      *            refreshed. If null, all nodes static flows will be refreshed.
1864      */
1865     private void refreshClusterStaticFlowsStatus(Node node) {
1866         // Refresh cluster cache
1867         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1868             if (node == null || entry.getValue().getNode().equals(node)) {
1869                 staticFlows.put(entry.getKey(), entry.getValue());
1870             }
1871         }
1872     }
1873
1874     /**
1875      * Uninstall all the non-internal Flow Entries present in the software view.
1876      * If requested, a copy of each original flow entry will be stored in the
1877      * inactive list so that it can be re-applied when needed (This is typically
1878      * the case when running in the default container and controller moved to
1879      * container mode)
1880      *
1881      * @param preserveFlowEntries
1882      *            if true, a copy of each original entry is stored in the
1883      *            inactive list
1884      */
1885     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1886         log.info("Uninstalling all non-internal flows");
1887
1888         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1889
1890         // Store entries / create target list
1891         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1892             FlowEntryInstall flowEntries = mapEntry.getValue();
1893             // Skip internal generated static flows
1894             if (!flowEntries.isInternal()) {
1895                 toRemove.add(flowEntries);
1896                 // Store the original entries if requested
1897                 if (preserveFlowEntries) {
1898                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1899                 }
1900             }
1901         }
1902
1903         // Now remove the entries
1904         for (FlowEntryInstall flowEntryHw : toRemove) {
1905             Status status = this.removeEntryInternal(flowEntryHw, false);
1906             if (!status.isSuccess()) {
1907                 log.warn("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1908             }
1909         }
1910     }
1911
1912     /**
1913      * Re-install all the Flow Entries present in the inactive list The inactive
1914      * list will be empty at the end of this call This function is called on the
1915      * default container instance of FRM only when the last container is deleted
1916      */
1917     private void reinstallAllFlowEntries() {
1918         log.info("Reinstalling all inactive flows");
1919
1920         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1921             this.addEntry(flowEntry, false);
1922         }
1923
1924         // Empty inactive list in any case
1925         inactiveFlows.clear();
1926     }
1927
1928     @Override
1929     public List<FlowConfig> getStaticFlows() {
1930         return getStaticFlowsOrderedList(staticFlows, staticFlowsOrdinal.get(0).intValue());
1931     }
1932
1933     // TODO: need to come out with a better algorithm for maintaining the order
1934     // of the configuration entries
1935     // with actual one, index associated to deleted entries cannot be reused and
1936     // map grows...
1937     private List<FlowConfig> getStaticFlowsOrderedList(ConcurrentMap<Integer, FlowConfig> flowMap, int maxKey) {
1938         List<FlowConfig> orderedList = new ArrayList<FlowConfig>();
1939         for (int i = 0; i <= maxKey; i++) {
1940             FlowConfig entry = flowMap.get(i);
1941             if (entry != null) {
1942                 orderedList.add(entry);
1943             }
1944         }
1945         return orderedList;
1946     }
1947
1948     @Override
1949     public FlowConfig getStaticFlow(String name, Node node) {
1950         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1951             if (entry.getValue().isByNameAndNodeIdEqual(name, node)) {
1952                 return entry.getValue();
1953             }
1954         }
1955         return null;
1956     }
1957
1958     @Override
1959     public List<FlowConfig> getStaticFlows(Node node) {
1960         List<FlowConfig> list = new ArrayList<FlowConfig>();
1961         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1962             if (entry.getValue().onNode(node)) {
1963                 list.add(entry.getValue());
1964             }
1965         }
1966         return list;
1967     }
1968
1969     @Override
1970     public List<String> getStaticFlowNamesForNode(Node node) {
1971         List<String> list = new ArrayList<String>();
1972         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1973             if (entry.getValue().onNode(node)) {
1974                 list.add(entry.getValue().getName());
1975             }
1976         }
1977         return list;
1978     }
1979
1980     @Override
1981     public List<Node> getListNodeWithConfiguredFlows() {
1982         Set<Node> set = new HashSet<Node>();
1983         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1984             set.add(entry.getValue().getNode());
1985         }
1986         return new ArrayList<Node>(set);
1987     }
1988
1989     @SuppressWarnings("unchecked")
1990     private void loadFlowConfiguration() {
1991         ObjectReader objReader = new ObjectReader();
1992         ConcurrentMap<Integer, FlowConfig> confList = (ConcurrentMap<Integer, FlowConfig>) objReader.read(this,
1993                 frmFileName);
1994
1995         ConcurrentMap<String, PortGroupConfig> pgConfig = (ConcurrentMap<String, PortGroupConfig>) objReader.read(this,
1996                 portGroupFileName);
1997
1998         if (pgConfig != null) {
1999             for (ConcurrentMap.Entry<String, PortGroupConfig> entry : pgConfig.entrySet()) {
2000                 addPortGroupConfig(entry.getKey(), entry.getValue().getMatchString(), true);
2001             }
2002         }
2003
2004         if (confList == null) {
2005             return;
2006         }
2007
2008         int maxKey = 0;
2009         for (Integer key : confList.keySet()) {
2010             if (key.intValue() > maxKey) {
2011                 maxKey = key.intValue();
2012             }
2013         }
2014
2015         for (FlowConfig conf : getStaticFlowsOrderedList(confList, maxKey)) {
2016             addStaticFlowInternal(conf, true);
2017         }
2018     }
2019
2020     @Override
2021     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2022         return ois.readObject();
2023     }
2024
2025     @Override
2026     public Status saveConfig() {
2027         return saveConfigInternal();
2028     }
2029
2030     private Status saveConfigInternal() {
2031         ObjectWriter objWriter = new ObjectWriter();
2032         ConcurrentMap<Integer, FlowConfig> nonDynamicFlows = new ConcurrentHashMap<Integer, FlowConfig>();
2033         for (Integer ordinal : staticFlows.keySet()) {
2034             FlowConfig config = staticFlows.get(ordinal);
2035             // Do not save dynamic and controller generated static flows
2036             if (config.isDynamic() || config.isInternalFlow()) {
2037                 continue;
2038             }
2039             nonDynamicFlows.put(ordinal, config);
2040         }
2041         objWriter.write(nonDynamicFlows, frmFileName);
2042         objWriter.write(new ConcurrentHashMap<String, PortGroupConfig>(portGroupConfigs), portGroupFileName);
2043         return new Status(StatusCode.SUCCESS, null);
2044     }
2045
2046     @Override
2047     public void subnetNotify(Subnet sub, boolean add) {
2048     }
2049
2050     private void installImplicitARPReplyPunt(Node node) {
2051
2052         if (node == null) {
2053             return;
2054         }
2055
2056         List<String> puntAction = new ArrayList<String>();
2057         puntAction.add(ActionType.CONTROLLER.toString());
2058
2059         FlowConfig allowARP = new FlowConfig();
2060         allowARP.setInstallInHw(true);
2061         allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP Reply" + FlowConfig.INTERNALSTATICFLOWEND);
2062         allowARP.setPriority("500");
2063         allowARP.setNode(node);
2064         allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue()).toUpperCase());
2065         allowARP.setDstMac(HexEncode.bytesToHexString(switchManager.getControllerMAC()));
2066         allowARP.setActions(puntAction);
2067         addStaticFlowInternal(allowARP, true); // skip validation on internal static flow name
2068     }
2069
2070     /**
2071      * (non-Javadoc)
2072      *
2073      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2074      *      boolean)
2075      *
2076      *      This method can be called from within the OSGi framework context,
2077      *      given the programming operation can take sometime, it not good
2078      *      pratice to have in it's context operations that can take time,
2079      *      hence moving off to a different thread for async processing.
2080      */
2081     private ExecutorService executor;
2082     @Override
2083     public void modeChangeNotify(final Node node, final boolean proactive) {
2084         Callable<Status> modeChangeCallable = new Callable<Status>() {
2085             @Override
2086             public Status call() throws Exception {
2087                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2088
2089                 List<String> puntAction = new ArrayList<String>();
2090                 puntAction.add(ActionType.CONTROLLER.toString());
2091
2092                 FlowConfig allowARP = new FlowConfig();
2093                 allowARP.setInstallInHw(true);
2094                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2095                 allowARP.setPriority("1");
2096                 allowARP.setNode(node);
2097                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2098                         .toUpperCase());
2099                 allowARP.setActions(puntAction);
2100                 defaultConfigs.add(allowARP);
2101
2102                 FlowConfig allowLLDP = new FlowConfig();
2103                 allowLLDP.setInstallInHw(true);
2104                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2105                 allowLLDP.setPriority("1");
2106                 allowLLDP.setNode(node);
2107                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2108                         .toUpperCase());
2109                 allowLLDP.setActions(puntAction);
2110                 defaultConfigs.add(allowLLDP);
2111
2112                 List<String> dropAction = new ArrayList<String>();
2113                 dropAction.add(ActionType.DROP.toString());
2114
2115                 FlowConfig dropAllConfig = new FlowConfig();
2116                 dropAllConfig.setInstallInHw(true);
2117                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2118                         + FlowConfig.INTERNALSTATICFLOWEND);
2119                 dropAllConfig.setPriority("0");
2120                 dropAllConfig.setNode(node);
2121                 dropAllConfig.setActions(dropAction);
2122                 defaultConfigs.add(dropAllConfig);
2123
2124                 log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2125                 for (FlowConfig fc : defaultConfigs) {
2126                     Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2127                     if (status.isSuccess()) {
2128                         log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2129                     } else {
2130                         log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2131                                 fc.getName());
2132                     }
2133                 }
2134                 return new Status(StatusCode.SUCCESS);
2135             }
2136         };
2137
2138         /*
2139          * Execute the work outside the caller context, this could be an
2140          * expensive operation and we don't want to block the caller for it.
2141          */
2142         this.executor.submit(modeChangeCallable);
2143     }
2144
2145     /**
2146      * Remove from the databases all the flows installed on the node
2147      *
2148      * @param node
2149      */
2150     private void cleanDatabaseForNode(Node node) {
2151         log.info("Cleaning Flow database for Node {}", node);
2152         if (nodeFlows.containsKey(node)) {
2153             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2154
2155             for (FlowEntryInstall entry : toRemove) {
2156                 updateLocalDatabase(entry, false);
2157             }
2158         }
2159     }
2160
2161     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2162         if (nc == null) {
2163             return false;
2164         }
2165
2166         Match match = flow.getMatch();
2167         if (match.isPresent(MatchType.IN_PORT)) {
2168             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2169             if (matchPort.equals(nc)) {
2170                 return true;
2171             }
2172         }
2173         List<Action> actionsList = flow.getActions();
2174         if (actionsList != null) {
2175             for (Action action : actionsList) {
2176                 if (action instanceof Output) {
2177                     NodeConnector actionPort = ((Output) action).getPort();
2178                     if (actionPort.equals(nc)) {
2179                         return true;
2180                     }
2181                 }
2182             }
2183         }
2184         return false;
2185     }
2186
2187     @Override
2188     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2189         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2190     }
2191
2192     @Override
2193     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2194         boolean updateStaticFlowCluster = false;
2195
2196         switch (type) {
2197         case ADDED:
2198             break;
2199         case CHANGED:
2200             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2201             if (config != null) {
2202                 switch (config.getValue()) {
2203                 case Config.ADMIN_DOWN:
2204                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2205                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2206                     break;
2207                 case Config.ADMIN_UP:
2208                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2209                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2210                     break;
2211                 case Config.ADMIN_UNDEF:
2212                     break;
2213                 default:
2214                 }
2215             }
2216             break;
2217         case REMOVED:
2218             // This is the case where a switch port is removed from the SDN agent space
2219             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2220             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2221             break;
2222         default:
2223
2224         }
2225
2226         if (updateStaticFlowCluster) {
2227             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2228         }
2229     }
2230
2231     /*
2232      * It goes through the static flows configuration, it identifies the ones
2233      * which have the specified node connector as input or output port and
2234      * install them on the network node if they are marked to be installed in
2235      * hardware and their status shows they were not installed yet
2236      */
2237     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2238         boolean updated = false;
2239         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2240         for (FlowConfig flowConfig : flowConfigForNode) {
2241             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2242                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(SUCCESS)) {
2243                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2244                     if (!status.isSuccess()) {
2245                         flowConfig.setStatus(status.getDescription());
2246                     } else {
2247                         flowConfig.setStatus(SUCCESS);
2248                     }
2249                     updated = true;
2250                 }
2251             }
2252         }
2253         return updated;
2254     }
2255
2256     /*
2257      * Remove from the network node all the flows which have the specified node
2258      * connector as input or output port. If any of the flow entry is a static
2259      * flow, it updates the correspondent configuration.
2260      */
2261     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2262         boolean updated = false;
2263         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2264         if (nodeFlowEntries == null) {
2265             return updated;
2266         }
2267         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2268             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2269                 Status status = this.removeEntryInternal(fei, true);
2270                 if (!status.isSuccess()) {
2271                     continue;
2272                 }
2273                 /*
2274                  * If the flow entry is a static flow, then update its
2275                  * configuration
2276                  */
2277                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2278                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2279                     if (flowConfig != null) {
2280                         flowConfig.setStatus(PORTREMOVED);
2281                         updated = true;
2282                     }
2283                 }
2284             }
2285         }
2286         return updated;
2287     }
2288
2289     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2290         FlowConfig derivedFlow = new FlowConfig(original);
2291         derivedFlow.setDynamic(true);
2292         derivedFlow.setPortGroup(null);
2293         derivedFlow.setName(original.getName() + "_" + configName + "_" + port);
2294         derivedFlow.setIngressPort(port + "");
2295         return derivedFlow;
2296     }
2297
2298     private void addPortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2299         for (FlowConfig staticFlow : staticFlows.values()) {
2300             if (staticFlow.getPortGroup() == null) {
2301                 continue;
2302             }
2303             if ((staticFlow.getNode().equals(node)) && (staticFlow.getPortGroup().equals(config.getName()))) {
2304                 for (Short port : data.getPorts()) {
2305                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2306                     addStaticFlowInternal(derivedFlow, false);
2307                 }
2308             }
2309         }
2310     }
2311
2312     private void removePortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2313         for (FlowConfig staticFlow : staticFlows.values()) {
2314             if (staticFlow.getPortGroup() == null) {
2315                 continue;
2316             }
2317             if (staticFlow.getNode().equals(node) && staticFlow.getPortGroup().equals(config.getName())) {
2318                 for (Short port : data.getPorts()) {
2319                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2320                     removeStaticFlow(derivedFlow);
2321                 }
2322             }
2323         }
2324     }
2325
2326     @Override
2327     public void portGroupChanged(PortGroupConfig config, Map<Node, PortGroup> data, boolean add) {
2328         log.info("PortGroup Changed for: {} Data: {}", config, portGroupData);
2329         Map<Node, PortGroup> existingData = portGroupData.get(config);
2330         if (existingData != null) {
2331             for (Map.Entry<Node, PortGroup> entry : data.entrySet()) {
2332                 PortGroup existingPortGroup = existingData.get(entry.getKey());
2333                 if (existingPortGroup == null) {
2334                     if (add) {
2335                         existingData.put(entry.getKey(), entry.getValue());
2336                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2337                     }
2338                 } else {
2339                     if (add) {
2340                         existingPortGroup.getPorts().addAll(entry.getValue().getPorts());
2341                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2342                     } else {
2343                         existingPortGroup.getPorts().removeAll(entry.getValue().getPorts());
2344                         removePortGroupFlows(config, entry.getKey(), entry.getValue());
2345                     }
2346                 }
2347             }
2348         } else {
2349             if (add) {
2350                 portGroupData.put(config, data);
2351                 for (Node swid : data.keySet()) {
2352                     addPortGroupFlows(config, swid, data.get(swid));
2353                 }
2354             }
2355         }
2356     }
2357
2358     @Override
2359     public boolean addPortGroupConfig(String name, String regex, boolean restore) {
2360         PortGroupConfig config = portGroupConfigs.get(name);
2361         if (config != null) {
2362             return false;
2363         }
2364
2365         if ((portGroupProvider == null) && !restore) {
2366             return false;
2367         }
2368         if ((portGroupProvider != null) && (!portGroupProvider.isMatchCriteriaSupported(regex))) {
2369             return false;
2370         }
2371
2372         config = new PortGroupConfig(name, regex);
2373         portGroupConfigs.put(name, config);
2374         if (portGroupProvider != null) {
2375             portGroupProvider.createPortGroupConfig(config);
2376         }
2377         return true;
2378     }
2379
2380     @Override
2381     public boolean delPortGroupConfig(String name) {
2382         PortGroupConfig config = portGroupConfigs.get(name);
2383         if (config == null) {
2384             return false;
2385         }
2386
2387         if (portGroupProvider != null) {
2388             portGroupProvider.deletePortGroupConfig(config);
2389         }
2390         portGroupConfigs.remove(name);
2391         return true;
2392     }
2393
2394     private void usePortGroupConfig(String name) {
2395         PortGroupConfig config = portGroupConfigs.get(name);
2396         if (config == null) {
2397             return;
2398         }
2399         if (portGroupProvider != null) {
2400             Map<Node, PortGroup> data = portGroupProvider.getPortGroupData(config);
2401             portGroupData.put(config, data);
2402         }
2403     }
2404
2405     @Override
2406     public Map<String, PortGroupConfig> getPortGroupConfigs() {
2407         return portGroupConfigs;
2408     }
2409
2410     public boolean isPortGroupSupported() {
2411         if (portGroupProvider == null) {
2412             return false;
2413         }
2414         return true;
2415     }
2416
2417     public void setIContainer(IContainer s) {
2418         this.container = s;
2419     }
2420
2421     public void unsetIContainer(IContainer s) {
2422         if (this.container == s) {
2423             this.container = null;
2424         }
2425     }
2426
2427     @Override
2428     public PortGroupProvider getPortGroupProvider() {
2429         return portGroupProvider;
2430     }
2431
2432     public void unsetPortGroupProvider(PortGroupProvider portGroupProvider) {
2433         this.portGroupProvider = null;
2434     }
2435
2436     public void setPortGroupProvider(PortGroupProvider portGroupProvider) {
2437         this.portGroupProvider = portGroupProvider;
2438         portGroupProvider.registerPortGroupChange(this);
2439         for (PortGroupConfig config : portGroupConfigs.values()) {
2440             portGroupProvider.createPortGroupConfig(config);
2441         }
2442     }
2443
2444     public void setFrmAware(IForwardingRulesManagerAware obj) {
2445         this.frmAware.add(obj);
2446     }
2447
2448     public void unsetFrmAware(IForwardingRulesManagerAware obj) {
2449         this.frmAware.remove(obj);
2450     }
2451
2452     void setClusterContainerService(IClusterContainerServices s) {
2453         log.debug("Cluster Service set");
2454         this.clusterContainerService = s;
2455     }
2456
2457     void unsetClusterContainerService(IClusterContainerServices s) {
2458         if (this.clusterContainerService == s) {
2459             log.debug("Cluster Service removed!");
2460             this.clusterContainerService = null;
2461         }
2462     }
2463
2464     private String getContainerName() {
2465         if (container == null) {
2466             return GlobalConstants.DEFAULT.toString();
2467         }
2468         return container.getName();
2469     }
2470
2471     /**
2472      * Function called by the dependency manager when all the required
2473      * dependencies are satisfied
2474      *
2475      */
2476     void init() {
2477         frmFileName = GlobalConstants.STARTUPHOME.toString() + "frm_staticflows_" + this.getContainerName() + ".conf";
2478         portGroupFileName = GlobalConstants.STARTUPHOME.toString() + "portgroup_" + this.getContainerName() + ".conf";
2479
2480         inContainerMode = false;
2481
2482         if (portGroupProvider != null) {
2483             portGroupProvider.registerPortGroupChange(this);
2484         }
2485
2486         cacheStartup();
2487
2488         registerWithOSGIConsole();
2489
2490         /*
2491          * If we are not the first cluster node to come up, do not initialize
2492          * the static flow entries ordinal
2493          */
2494         if (staticFlowsOrdinal.size() == 0) {
2495             staticFlowsOrdinal.put(0, Integer.valueOf(0));
2496         }
2497
2498         pendingEvents = new LinkedBlockingQueue<FRMEvent>();
2499
2500         // Initialize the event handler thread
2501         frmEventHandler = new Thread(new Runnable() {
2502             @Override
2503             public void run() {
2504                 while (!stopping) {
2505                     try {
2506                         FRMEvent event = pendingEvents.take();
2507                         if (event == null) {
2508                             log.warn("Dequeued null event");
2509                             continue;
2510                         }
2511                         if (event instanceof NodeUpdateEvent) {
2512                             NodeUpdateEvent update = (NodeUpdateEvent) event;
2513                             Node node = update.getNode();
2514                             switch (update.getUpdateType()) {
2515                             case ADDED:
2516                                 addStaticFlowsToSwitch(node);
2517                                 break;
2518                             case REMOVED:
2519                                 cleanDatabaseForNode(node);
2520                                 updateStaticFlowConfigsOnNodeDown(node);
2521                                 break;
2522                             default:
2523                             }
2524                         } else if (event instanceof ErrorReportedEvent) {
2525                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
2526                             processErrorEvent(errEvent);
2527                         } else if (event instanceof WorkOrderEvent) {
2528                             /*
2529                              * Take care of handling the remote Work request
2530                              */
2531                             WorkOrderEvent work = (WorkOrderEvent) event;
2532                             FlowEntryDistributionOrder fe = work.getFe();
2533                             if (fe != null) {
2534                                 logsync.trace("Executing the workOrder {}", fe);
2535                                 Status gotStatus = null;
2536                                 FlowEntryInstall feiCurrent = fe.getEntry();
2537                                 FlowEntryInstall feiNew = workOrder.get(fe.getEntry());
2538                                 switch (fe.getUpType()) {
2539                                 case ADDED:
2540                                     /*
2541                                      * TODO: Not still sure how to handle the
2542                                      * sync entries
2543                                      */
2544                                     gotStatus = addEntriesInternal(feiCurrent, true);
2545                                     break;
2546                                 case CHANGED:
2547                                     gotStatus = modifyEntryInternal(feiCurrent, feiNew, true);
2548                                     break;
2549                                 case REMOVED:
2550                                     gotStatus = removeEntryInternal(feiCurrent, true);
2551                                     break;
2552                                 }
2553                                 // Remove the Order
2554                                 workOrder.remove(fe);
2555                                 logsync.trace(
2556                                         "The workOrder has been executed and now the status is being returned {}", fe);
2557                                 // Place the status
2558                                 workStatus.put(fe, gotStatus);
2559                             } else {
2560                                 log.warn("Not expected null WorkOrder", work);
2561                             }
2562                         } else if (event instanceof WorkStatusCleanup) {
2563                             /*
2564                              * Take care of handling the remote Work request
2565                              */
2566                             WorkStatusCleanup work = (WorkStatusCleanup) event;
2567                             FlowEntryDistributionOrder fe = work.getFe();
2568                             if (fe != null) {
2569                                 logsync.trace("The workStatus {} is being removed", fe);
2570                                 workStatus.remove(fe);
2571                             } else {
2572                                 log.warn("Not expected null WorkStatus", work);
2573                             }
2574                         }  else if (event instanceof ContainerFlowChangeEvent) {
2575                             /*
2576                              * Whether it is an addition or removal, we have to
2577                              * recompute the merged flows entries taking into
2578                              * account all the current container flows because
2579                              * flow merging is not an injective function
2580                              */
2581                             updateFlowsContainerFlow();
2582                         } else {
2583                             log.warn("Dequeued unknown event {}", event.getClass()
2584                                     .getSimpleName());
2585                         }
2586                     } catch (InterruptedException e) {
2587                         // clear pending events
2588                         pendingEvents.clear();
2589                     }
2590                 }
2591             }
2592         }, "FRM EventHandler Collector");
2593     }
2594
2595     /**
2596      * Function called by the dependency manager when at least one dependency
2597      * become unsatisfied or when the component is shutting down because for
2598      * example bundle is being stopped.
2599      *
2600      */
2601     void destroy() {
2602         // Interrupt the thread
2603         frmEventHandler.interrupt();
2604         // Clear the pendingEvents queue
2605         pendingEvents.clear();
2606         frmAware.clear();
2607         workMonitor.clear();
2608     }
2609
2610     /**
2611      * Function called by dependency manager after "init ()" is called and after
2612      * the services provided by the class are registered in the service registry
2613      *
2614      */
2615     void start() {
2616         // Initialize graceful stop flag
2617         stopping = false;
2618
2619         // Allocate the executor service
2620         this.executor = Executors.newSingleThreadExecutor();
2621
2622         // Start event handler thread
2623         frmEventHandler.start();
2624
2625         /*
2626          * Read startup and build database if we have not already gotten the
2627          * configurations synced from another node
2628          */
2629         if (staticFlows.isEmpty()) {
2630             loadFlowConfiguration();
2631         }
2632     }
2633
2634     /**
2635      * Function called by the dependency manager before the services exported by
2636      * the component are unregistered, this will be followed by a "destroy ()"
2637      * calls
2638      */
2639     void stop() {
2640         stopping = true;
2641         uninstallAllFlowEntries(false);
2642         // Shutdown executor
2643         this.executor.shutdownNow();
2644     }
2645
2646     public void setFlowProgrammerService(IFlowProgrammerService service) {
2647         this.programmer = service;
2648     }
2649
2650     public void unsetFlowProgrammerService(IFlowProgrammerService service) {
2651         if (this.programmer == service) {
2652             this.programmer = null;
2653         }
2654     }
2655
2656     public void setSwitchManager(ISwitchManager switchManager) {
2657         this.switchManager = switchManager;
2658     }
2659
2660     public void unsetSwitchManager(ISwitchManager switchManager) {
2661         if (this.switchManager == switchManager) {
2662             this.switchManager = null;
2663         }
2664     }
2665
2666     @Override
2667     public void tagUpdated(String containerName, Node n, short oldTag, short newTag, UpdateType t) {
2668         if (!container.getName().equals(containerName)) {
2669             return;
2670         }
2671     }
2672
2673     @Override
2674     public void containerFlowUpdated(String containerName, ContainerFlow previous, ContainerFlow current, UpdateType t) {
2675         if (!container.getName().equals(containerName)) {
2676             return;
2677         }
2678         log.trace("Container {}: Updating installed flows because of container flow change: {} {}",
2679                 container.getName(), t, current);
2680         ContainerFlowChangeEvent ev = new ContainerFlowChangeEvent(previous, current, t);
2681         pendingEvents.offer(ev);
2682     }
2683
2684     @Override
2685     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
2686         if (!container.getName().equals(containerName)) {
2687             return;
2688         }
2689
2690         boolean updateStaticFlowCluster = false;
2691
2692         switch (t) {
2693         case REMOVED:
2694             log.trace("Port {} was removed from container: uninstalling interested flows", nc);
2695             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nc);
2696             break;
2697         case ADDED:
2698             log.trace("Port {} was added to container: reinstall interested flows", nc);
2699             updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nc);
2700
2701             break;
2702         case CHANGED:
2703             break;
2704         default:
2705         }
2706
2707         if (updateStaticFlowCluster) {
2708             refreshClusterStaticFlowsStatus(nc.getNode());
2709         }
2710     }
2711
2712     @Override
2713     public void containerModeUpdated(UpdateType update) {
2714         // Only default container instance reacts on this event
2715         if (!container.getName().equals(GlobalConstants.DEFAULT.toString())) {
2716             return;
2717         }
2718         switch (update) {
2719         case ADDED:
2720             /*
2721              * Controller is moving to container mode. We are in the default
2722              * container context, we need to remove all our non-internal flows
2723              * to prevent any container isolation breakage. We also need to
2724              * preserve our flow so that they can be re-installed if we move
2725              * back to non container mode (no containers).
2726              */
2727             this.inContainerMode = true;
2728             this.uninstallAllFlowEntries(true);
2729             break;
2730         case REMOVED:
2731             this.inContainerMode = false;
2732             this.reinstallAllFlowEntries();
2733             break;
2734         default:
2735         }
2736
2737         // Update our configuration DB
2738         updateStaticFlowConfigsOnContainerModeChange(update);
2739     }
2740
2741     protected abstract class FRMEvent {
2742
2743     }
2744
2745     private class NodeUpdateEvent extends FRMEvent {
2746         private final Node node;
2747         private final UpdateType update;
2748
2749         public NodeUpdateEvent(UpdateType update, Node node) {
2750             this.update = update;
2751             this.node = node;
2752         }
2753
2754         public UpdateType getUpdateType() {
2755             return update;
2756         }
2757
2758         public Node getNode() {
2759             return node;
2760         }
2761     }
2762
2763     private class ErrorReportedEvent extends FRMEvent {
2764         private final long rid;
2765         private final Node node;
2766         private final Object error;
2767
2768         public ErrorReportedEvent(long rid, Node node, Object error) {
2769             this.rid = rid;
2770             this.node = node;
2771             this.error = error;
2772         }
2773
2774         public long getRequestId() {
2775             return rid;
2776         }
2777
2778         public Object getError() {
2779             return error;
2780         }
2781
2782         public Node getNode() {
2783             return node;
2784         }
2785     }
2786
2787     private class WorkOrderEvent extends FRMEvent {
2788         private FlowEntryDistributionOrder fe;
2789         private FlowEntryInstall newEntry;
2790
2791         /**
2792          * @param fe
2793          * @param newEntry
2794          */
2795         WorkOrderEvent(FlowEntryDistributionOrder fe, FlowEntryInstall newEntry) {
2796             this.fe = fe;
2797             this.newEntry = newEntry;
2798         }
2799
2800         /**
2801          * @return the fe
2802          */
2803         public FlowEntryDistributionOrder getFe() {
2804             return fe;
2805         }
2806
2807         /**
2808          * @return the newEntry
2809          */
2810         public FlowEntryInstall getNewEntry() {
2811             return newEntry;
2812         }
2813     }
2814     private class ContainerFlowChangeEvent extends FRMEvent {
2815         private final ContainerFlow previous;
2816         private final ContainerFlow current;
2817         private final UpdateType type;
2818
2819         public ContainerFlowChangeEvent(ContainerFlow previous, ContainerFlow current, UpdateType type) {
2820             this.previous = previous;
2821             this.current = current;
2822             this.type = type;
2823         }
2824
2825         public ContainerFlow getPrevious() {
2826             return this.previous;
2827         }
2828
2829         public ContainerFlow getCurrent() {
2830             return this.current;
2831         }
2832
2833         public UpdateType getType() {
2834             return this.type;
2835         }
2836     }
2837
2838
2839     private class WorkStatusCleanup extends FRMEvent {
2840         private FlowEntryDistributionOrder fe;
2841
2842         /**
2843          * @param fe
2844          */
2845         WorkStatusCleanup(FlowEntryDistributionOrder fe) {
2846             this.fe = fe;
2847         }
2848
2849         /**
2850          * @return the fe
2851          */
2852         public FlowEntryDistributionOrder getFe() {
2853             return fe;
2854         }
2855     }
2856
2857     /*
2858      * OSGI COMMANDS
2859      */
2860     @Override
2861     public String getHelp() {
2862         StringBuffer help = new StringBuffer();
2863         help.append("---FRM Matrix Application---\n");
2864         help.append("\t printMatrixData        - Prints the Matrix Configs\n");
2865         help.append("\t addMatrixConfig <name> <regex>\n");
2866         help.append("\t delMatrixConfig <name>\n");
2867         help.append("\t useMatrixConfig <name>\n");
2868         return help.toString();
2869     }
2870
2871     public void _printMatrixData(CommandInterpreter ci) {
2872         ci.println("Configs : ");
2873         ci.println("---------");
2874         ci.println(portGroupConfigs);
2875
2876         ci.println("Data : ");
2877         ci.println("------");
2878         ci.println(portGroupData);
2879     }
2880
2881     public void _addMatrixConfig(CommandInterpreter ci) {
2882         String name = ci.nextArgument();
2883         String regex = ci.nextArgument();
2884         addPortGroupConfig(name, regex, false);
2885     }
2886
2887     public void _delMatrixConfig(CommandInterpreter ci) {
2888         String name = ci.nextArgument();
2889         delPortGroupConfig(name);
2890     }
2891
2892     public void _useMatrixConfig(CommandInterpreter ci) {
2893         String name = ci.nextArgument();
2894         usePortGroupConfig(name);
2895     }
2896
2897     public void _arpPunt(CommandInterpreter ci) {
2898         String switchId = ci.nextArgument();
2899         long swid = HexEncode.stringToLong(switchId);
2900         Node node = NodeCreator.createOFNode(swid);
2901         installImplicitARPReplyPunt(node);
2902     }
2903
2904     public void _frmaddflow(CommandInterpreter ci) throws UnknownHostException {
2905         Node node = null;
2906         String nodeId = ci.nextArgument();
2907         if (nodeId == null) {
2908             ci.print("Node id not specified");
2909             return;
2910         }
2911         try {
2912             node = NodeCreator.createOFNode(Long.valueOf(nodeId));
2913         } catch (NumberFormatException e) {
2914             ci.print("Node id not a number");
2915             return;
2916         }
2917         ci.println(this.programmer.addFlow(node, getSampleFlow(node)));
2918     }
2919
2920     public void _frmremoveflow(CommandInterpreter ci) throws UnknownHostException {
2921         Node node = null;
2922         String nodeId = ci.nextArgument();
2923         if (nodeId == null) {
2924             ci.print("Node id not specified");
2925             return;
2926         }
2927         try {
2928             node = NodeCreator.createOFNode(Long.valueOf(nodeId));
2929         } catch (NumberFormatException e) {
2930             ci.print("Node id not a number");
2931             return;
2932         }
2933         ci.println(this.programmer.removeFlow(node, getSampleFlow(node)));
2934     }
2935
2936     private Flow getSampleFlow(Node node) throws UnknownHostException {
2937         NodeConnector port = NodeConnectorCreator.createOFNodeConnector((short) 24, node);
2938         NodeConnector oport = NodeConnectorCreator.createOFNodeConnector((short) 30, node);
2939         byte srcMac[] = { (byte) 0x12, (byte) 0x34, (byte) 0x56, (byte) 0x78, (byte) 0x9a, (byte) 0xbc };
2940         byte dstMac[] = { (byte) 0x1a, (byte) 0x2b, (byte) 0x3c, (byte) 0x4d, (byte) 0x5e, (byte) 0x6f };
2941         InetAddress srcIP = InetAddress.getByName("172.28.30.50");
2942         InetAddress dstIP = InetAddress.getByName("171.71.9.52");
2943         InetAddress ipMask = InetAddress.getByName("255.255.255.0");
2944         InetAddress ipMask2 = InetAddress.getByName("255.0.0.0");
2945         short ethertype = EtherTypes.IPv4.shortValue();
2946         short vlan = (short) 27;
2947         byte vlanPr = 3;
2948         Byte tos = 4;
2949         byte proto = IPProtocols.TCP.byteValue();
2950         short src = (short) 55000;
2951         short dst = 80;
2952
2953         /*
2954          * Create a SAL Flow aFlow
2955          */
2956         Match match = new Match();
2957         match.setField(MatchType.IN_PORT, port);
2958         match.setField(MatchType.DL_SRC, srcMac);
2959         match.setField(MatchType.DL_DST, dstMac);
2960         match.setField(MatchType.DL_TYPE, ethertype);
2961         match.setField(MatchType.DL_VLAN, vlan);
2962         match.setField(MatchType.DL_VLAN_PR, vlanPr);
2963         match.setField(MatchType.NW_SRC, srcIP, ipMask);
2964         match.setField(MatchType.NW_DST, dstIP, ipMask2);
2965         match.setField(MatchType.NW_TOS, tos);
2966         match.setField(MatchType.NW_PROTO, proto);
2967         match.setField(MatchType.TP_SRC, src);
2968         match.setField(MatchType.TP_DST, dst);
2969
2970         List<Action> actions = new ArrayList<Action>();
2971         actions.add(new Output(oport));
2972         actions.add(new PopVlan());
2973         actions.add(new Flood());
2974         actions.add(new Controller());
2975         return new Flow(match, actions);
2976     }
2977
2978     @Override
2979     public Status saveConfiguration() {
2980         return saveConfig();
2981     }
2982
2983     public void _frmNodeFlows(CommandInterpreter ci) {
2984         String nodeId = ci.nextArgument();
2985         Node node = Node.fromString(nodeId);
2986         if (node == null) {
2987             ci.println("frmNodeFlows <node> [verbose]");
2988             return;
2989         }
2990         boolean verbose = false;
2991         String verboseCheck = ci.nextArgument();
2992         if (verboseCheck != null) {
2993             verbose = verboseCheck.equals("true");
2994         }
2995
2996         if (!nodeFlows.containsKey(node)) {
2997             return;
2998         }
2999         // Dump per node database
3000         for (FlowEntryInstall entry : nodeFlows.get(node)) {
3001             if (!verbose) {
3002                 ci.println(node + " " + installedSwView.get(entry).getFlowName());
3003             } else {
3004                 ci.println(node + " " + installedSwView.get(entry).toString());
3005             }
3006         }
3007     }
3008
3009     public void _frmGroupFlows(CommandInterpreter ci) {
3010         String group = ci.nextArgument();
3011         if (group == null) {
3012             ci.println("frmGroupFlows <group> [verbose]");
3013             return;
3014         }
3015         boolean verbose = false;
3016         String verboseCheck = ci.nextArgument();
3017         if (verboseCheck != null) {
3018             verbose = verboseCheck.equalsIgnoreCase("true");
3019         }
3020
3021         if (!groupFlows.containsKey(group)) {
3022             return;
3023         }
3024         // Dump per node database
3025         ci.println("Group " + group + ":\n");
3026         for (FlowEntryInstall flowEntry : groupFlows.get(group)) {
3027             if (!verbose) {
3028                 ci.println(flowEntry.getNode() + " " + flowEntry.getFlowName());
3029             } else {
3030                 ci.println(flowEntry.getNode() + " " + flowEntry.toString());
3031             }
3032         }
3033     }
3034
3035     @Override
3036     public void flowRemoved(Node node, Flow flow) {
3037         log.trace("Received flow removed notification on {} for {}", node, flow);
3038
3039         // For flow entry identification, only node, match and priority matter
3040         FlowEntryInstall test = new FlowEntryInstall(new FlowEntry("", "", flow, node), null);
3041         FlowEntryInstall installedEntry = this.installedSwView.get(test);
3042         if (installedEntry == null) {
3043             log.trace("Entry is not known to us");
3044             return;
3045         }
3046
3047         // Update Static flow status
3048         Integer key = 0;
3049         FlowConfig target = null;
3050         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
3051             FlowConfig conf = entry.getValue();
3052             if (conf.isByNameAndNodeIdEqual(installedEntry.getFlowName(), node)) {
3053                 key = entry.getKey();
3054                 target = conf;
3055                 break;
3056             }
3057         }
3058         if (target != null) {
3059             // Update Configuration database
3060             target.toggleInstallation();
3061             target.setStatus(SUCCESS);
3062             staticFlows.put(key, target);
3063         }
3064
3065         // Update software views
3066         this.updateLocalDatabase(installedEntry, false);
3067     }
3068
3069     @Override
3070     public void flowErrorReported(Node node, long rid, Object err) {
3071         log.trace("Got error {} for message rid {} from node {}", new Object[] { err, rid, node });
3072         pendingEvents.offer(new ErrorReportedEvent(rid, node, err));
3073     }
3074
3075     private void processErrorEvent(ErrorReportedEvent event) {
3076         Node node = event.getNode();
3077         long rid = event.getRequestId();
3078         Object error = event.getError();
3079         String errorString = (error == null) ? "Not provided" : error.toString();
3080         /*
3081          * If this was for a flow install, remove the corresponding entry from
3082          * the software view. If it was a Looking for the rid going through the
3083          * software database. TODO: A more efficient rid <-> FlowEntryInstall
3084          * mapping will have to be added in future
3085          */
3086         FlowEntryInstall target = null;
3087         List<FlowEntryInstall> flowEntryInstallList = nodeFlows.get(node);
3088         // flowEntryInstallList could be null.
3089         // so check for it.
3090         if(flowEntryInstallList != null) {
3091             for (FlowEntryInstall index : flowEntryInstallList) {
3092                 FlowEntryInstall entry = installedSwView.get(index);
3093                 if(entry != null) {
3094                     if (entry.getRequestId() == rid) {
3095                         target = entry;
3096                         break;
3097                     }
3098                 }
3099             }
3100         }
3101         if (target != null) {
3102             // This was a flow install, update database
3103             this.updateLocalDatabase(target, false);
3104         }
3105
3106         // Notify listeners
3107         if (frmAware != null) {
3108             synchronized (frmAware) {
3109                 for (IForwardingRulesManagerAware frma : frmAware) {
3110                     try {
3111                         frma.requestFailed(rid, errorString);
3112                     } catch (Exception e) {
3113                         log.warn("Failed to notify {}", frma);
3114                     }
3115                 }
3116             }
3117         }
3118     }
3119
3120     @Override
3121     public Status solicitStatusResponse(Node node, boolean blocking) {
3122         Status rv = new Status(StatusCode.INTERNALERROR);
3123
3124         if (this.programmer != null) {
3125             if (blocking) {
3126                 rv = programmer.syncSendBarrierMessage(node);
3127             } else {
3128                 rv = programmer.asyncSendBarrierMessage(node);
3129             }
3130         }
3131
3132         return rv;
3133     }
3134
3135     public void unsetIConnectionManager(IConnectionManager s) {
3136         if (s == this.connectionManager) {
3137             this.connectionManager = null;
3138         }
3139     }
3140
3141     public void setIConnectionManager(IConnectionManager s) {
3142         this.connectionManager = s;
3143     }
3144
3145     @Override
3146     public void entryCreated(Object key, String cacheName, boolean originLocal) {
3147         /*
3148          * Do nothing
3149          */
3150     }
3151
3152     @Override
3153     public void entryUpdated(Object key, Object new_value, String cacheName, boolean originLocal) {
3154         if (originLocal) {
3155             /*
3156              * Local updates are of no interest
3157              */
3158             return;
3159         }
3160         if (cacheName.equals(WORKORDERCACHE)) {
3161             logsync.trace("Got a WorkOrderCacheUpdate for {}", key);
3162             /*
3163              * This is the case of one workOrder becoming available, so we need
3164              * to dispatch the work to the appropriate handler
3165              */
3166             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3167             FlowEntryInstall fei = fe.getEntry();
3168             if (fei == null) {
3169                 return;
3170             }
3171             Node n = fei.getNode();
3172             if (connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
3173                 logsync.trace("workOrder for fe {} processed locally", fe);
3174                 // I'm the controller in charge for the request, queue it for
3175                 // processing
3176                 pendingEvents.offer(new WorkOrderEvent(fe, (FlowEntryInstall) new_value));
3177             }
3178         } else if (cacheName.equals(WORKSTATUSCACHE)) {
3179             logsync.trace("Got a WorkStatusCacheUpdate for {}", key);
3180             /*
3181              * This is the case of one workOrder being completed and a status
3182              * returned
3183              */
3184             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3185             /*
3186              * Check if the order was initiated by this controller in that case
3187              * we need to actually look at the status returned
3188              */
3189             if (fe.getRequestorController()
3190                     .equals(clusterContainerService.getMyAddress())) {
3191                 FlowEntryDistributionOrderFutureTask fet = workMonitor.remove(fe);
3192                 if (fet != null) {
3193                     logsync.trace("workStatus response is for us {}", fe);
3194                     // Signal we got the status
3195                     fet.gotStatus(fe, workStatus.get(fe));
3196                     pendingEvents.offer(new WorkStatusCleanup(fe));
3197                 }
3198             }
3199         }
3200     }
3201
3202     @Override
3203     public void entryDeleted(Object key, String cacheName, boolean originLocal) {
3204         /*
3205          * Do nothing
3206          */
3207     }
3208 }