opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManager.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.forwardingrulesmanager.internal;
  10
  11 import java.io.FileNotFoundException;
  12 import java.io.IOException;
  13 import java.io.ObjectInputStream;
  14 import java.util.ArrayList;
  15 import java.util.Collections;
  16 import java.util.EnumSet;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Map.Entry;
  21 import java.util.Set;
  22 import java.util.concurrent.BlockingQueue;
  23 import java.util.concurrent.Callable;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentMap;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.ExecutorService;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30
  31 import org.opendaylight.controller.clustering.services.CacheConfigException;
  32 import org.opendaylight.controller.clustering.services.CacheExistException;
  33 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
  34 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
  35 import org.opendaylight.controller.clustering.services.IClusterServices;
  36 import org.opendaylight.controller.configuration.ConfigurationObject;
  37 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
  38 import org.opendaylight.controller.configuration.IConfigurationContainerService;
  39 import org.opendaylight.controller.connectionmanager.IConnectionManager;
  40 import org.opendaylight.controller.containermanager.IContainerManager;
  41 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
  42 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
  43 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
  44 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  45 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
  46 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
  47 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
  48 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
  49 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
  50 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
  51 import org.opendaylight.controller.sal.action.Action;
  52 import org.opendaylight.controller.sal.action.ActionType;
  53 import org.opendaylight.controller.sal.action.Output;
  54 import org.opendaylight.controller.sal.connection.ConnectionLocality;
  55 import org.opendaylight.controller.sal.core.Config;
  56 import org.opendaylight.controller.sal.core.ContainerFlow;
  57 import org.opendaylight.controller.sal.core.IContainer;
  58 import org.opendaylight.controller.sal.core.IContainerLocalListener;
  59 import org.opendaylight.controller.sal.core.Node;
  60 import org.opendaylight.controller.sal.core.NodeConnector;
  61 import org.opendaylight.controller.sal.core.Property;
  62 import org.opendaylight.controller.sal.core.UpdateType;
  63 import org.opendaylight.controller.sal.flowprogrammer.Flow;
  64 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
  65 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
  66 import org.opendaylight.controller.sal.match.Match;
  67 import org.opendaylight.controller.sal.match.MatchType;
  68 import org.opendaylight.controller.sal.utils.EtherTypes;
  69 import org.opendaylight.controller.sal.utils.GlobalConstants;
  70 import org.opendaylight.controller.sal.utils.IObjectReader;
  71 import org.opendaylight.controller.sal.utils.Status;
  72 import org.opendaylight.controller.sal.utils.StatusCode;
  73 import org.opendaylight.controller.switchmanager.IInventoryListener;
  74 import org.opendaylight.controller.switchmanager.ISwitchManager;
  75 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
  76 import org.opendaylight.controller.switchmanager.Subnet;
  77 import org.slf4j.Logger;
  78 import org.slf4j.LoggerFactory;
  79
  80 /**
  81  * Class that manages forwarding rule installation and removal per container of
  82  * the network. It also maintains the central repository of all the forwarding
  83  * rules installed on the network nodes.
  84  */
  85 public class ForwardingRulesManager implements
  86         IForwardingRulesManager,
  87         PortGroupChangeListener,
  88         IContainerLocalListener,
  89         ISwitchManagerAware,
  90         IConfigurationContainerAware,
  91         IInventoryListener,
  92         IObjectReader,
  93         ICacheUpdateAware<Object,Object>,
  94         IFlowProgrammerListener {
  95
  96     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
  97     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
  98     private static final String PORT_REMOVED = "Port removed";
  99     private static final String NODE_DOWN = "Node is Down";
 100     private static final String INVALID_FLOW_ENTRY = "Invalid FlowEntry";
 101     private static final String STATIC_FLOWS_FILE_NAME = "frm_staticflows.conf";
 102     private static final String PORT_GROUP_FILE_NAME = "portgroup.conf";
 103     private ConcurrentMap<Integer, FlowConfig> staticFlows;
 104     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
 105     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
 106     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
 107     private ConcurrentMap<String, Object> TSPolicies;
 108     private IContainerManager containerManager;
 109     private IConfigurationContainerService configurationService;
 110     private boolean inContainerMode; // being used by global instance only
 111     protected boolean stopping;
 112
 113     /*
 114      * Flow database. It's the software view of what was requested to install
 115      * and what is installed on the switch. It is indexed by the entry itself.
 116      * The entry's hashcode resumes the network node index, the flow's priority
 117      * and the flow's match. The value element is a class which contains the
 118      * flow entry pushed by the applications modules and the respective
 119      * container flow merged version. In absence of container flows, the two
 120      * flow entries are the same.
 121      */
 122     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
 123     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
 124     /*
 125      * Per node and per group indexing
 126      */
 127     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
 128     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
 129
 130     /*
 131      * Inactive flow list. This is for the global instance of FRM It will
 132      * contain all the flow entries which were installed on the global container
 133      * when the first container is created.
 134      */
 135     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
 136
 137     private IContainer container;
 138     private Set<IForwardingRulesManagerAware> frmAware =
 139         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
 140     private PortGroupProvider portGroupProvider;
 141     private IFlowProgrammerService programmer;
 142     private IClusterContainerServices clusterContainerService = null;
 143     private ISwitchManager switchManager;
 144     private Thread frmEventHandler;
 145     protected BlockingQueue<FRMEvent> pendingEvents;
 146
 147     // Distributes FRM programming in the cluster
 148     private IConnectionManager connectionManager;
 149
 150     /*
 151      * Name clustered caches used to support FRM entry distribution these are by
 152      * necessity non-transactional as long as need to be able to synchronize
 153      * states also while a transaction is in progress
 154      */
 155     static final String WORK_ORDER_CACHE = "frm.workOrder";
 156     static final String WORK_STATUS_CACHE = "frm.workStatus";
 157     static final String ORIGINAL_SW_VIEW_CACHE = "frm.originalSwView";
 158     static final String INSTALLED_SW_VIEW_CACHE = "frm.installedSwView";
 159
 160     /*
 161      * Data structure responsible for distributing the FlowEntryInstall requests
 162      * in the cluster. The key value is entry that is being either Installed or
 163      * Updated or Delete. The value field is the same of the key value in case
 164      * of Installation or Deletion, it's the new entry in case of Modification,
 165      * this because the clustering caches don't allow null values.
 166      *
 167      * The logic behind this data structure is that the controller that initiate
 168      * the request will place the order here, someone will pick it and then will
 169      * remove from this data structure because is being served.
 170      *
 171      * TODO: We need to have a way to cleanup this data structure if entries are
 172      * not picked by anyone, which is always a case can happen especially on
 173      * Node disconnect cases.
 174      */
 175     protected ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
 176
 177     /*
 178      * Data structure responsible for retrieving the results of the workOrder
 179      * submitted to the cluster.
 180      *
 181      * The logic behind this data structure is that the controller that has
 182      * executed the order will then place the result in workStatus signaling
 183      * that there was a success or a failure.
 184      *
 185      * TODO: The workStatus entries need to have a lifetime associated in case
 186      * of requestor controller leaving the cluster.
 187      */
 188     protected ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
 189
 190     /*
 191      * Local Map used to hold the Future which a caller can use to monitor for
 192      * completion
 193      */
 194     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
 195             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
 196
 197     /*
 198      * Max pool size for the executor
 199      */
 200     private static final int maxPoolSize = 10;
 201
 202     /**
 203      * @param e
 204      *            Entry being installed/updated/removed
 205      * @param u
 206      *            New entry will be placed after the update operation. Valid
 207      *            only for UpdateType.CHANGED, null for all the other cases
 208      * @param t
 209      *            Type of update
 210      * @return a Future object for monitoring the progress of the result, or
 211      *         null in case the processing should take place locally
 212      */
 213     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
 214             UpdateType t) {
 215         // A null entry it's an unexpected condition, anyway it's safe to keep
 216         // the handling local
 217         if (e == null) {
 218             return null;
 219         }
 220
 221         Node n = e.getNode();
 222         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
 223             // Create the work order and distribute it
 224             FlowEntryDistributionOrder fe =
 225                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
 226             // First create the monitor job
 227             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
 228             logsync.trace("Node {} not local so sending fe {}", n, fe);
 229             workMonitor.put(fe, ret);
 230             if (t.equals(UpdateType.CHANGED)) {
 231                 // Then distribute the work
 232                 workOrder.put(fe, u);
 233             } else {
 234                 // Then distribute the work
 235                 workOrder.put(fe, e);
 236             }
 237             logsync.trace("WorkOrder requested");
 238             // Now create an Handle to monitor the execution of the operation
 239             return ret;
 240         }
 241
 242         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
 243         return null;
 244     }
 245
 246     /**
 247      * Adds a flow entry onto the network node It runs various validity checks
 248      * and derive the final container flows merged entries that will be
 249      * attempted to be installed
 250      *
 251      * @param flowEntry
 252      *            the original flow entry application requested to add
 253      * @param async
 254      *            the flag indicating if this is a asynchronous request
 255      * @return the status of this request. In case of asynchronous call, it will
 256      *         contain the unique id assigned to this request
 257      */
 258     private Status addEntry(FlowEntry flowEntry, boolean async) {
 259
 260         // Sanity Check
 261         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 262             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 263             log.warn(logMsg, flowEntry);
 264             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 265         }
 266
 267         /*
 268          * Redundant Check: Check if the request is a redundant one from the
 269          * same application the flowEntry is equal to an existing one. Given we
 270          * do not have an application signature in the requested FlowEntry yet,
 271          * we are here detecting the above condition by comparing the flow
 272          * names, if set. If they are equal to the installed flow, most likely
 273          * this is a redundant installation request from the same application
 274          * and we can silently return success
 275          *
 276          * TODO: in future a sort of application reference list mechanism will
 277          * be added to the FlowEntry so that exact flow can be used by different
 278          * applications.
 279          */
 280         FlowEntry present = this.originalSwView.get(flowEntry);
 281         if (present != null) {
 282             boolean sameFlow = present.getFlow().equals(flowEntry.getFlow());
 283             boolean sameApp = present.getFlowName() != null && present.getFlowName().equals(flowEntry.getFlowName());
 284             if (sameFlow && sameApp) {
 285                 log.trace("Skipping redundant request for flow {} on node {}", flowEntry.getFlowName(),
 286                         flowEntry.getNode());
 287                 return new Status(StatusCode.SUCCESS, "Entry is already present");
 288             }
 289         }
 290
 291         /*
 292          * Derive the container flow merged entries to install In presence of N
 293          * container flows, we may end up with N different entries to install...
 294          */
 295         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 296
 297         // Container Flow conflict Check
 298         if (toInstallList.isEmpty()) {
 299             String msg = "Flow Entry conflicts with all Container Flows";
 300             String logMsg = msg + ": {}";
 301             log.warn(logMsg, flowEntry);
 302             return new Status(StatusCode.CONFLICT, msg);
 303         }
 304
 305         // Derive the list of entries good to be installed
 306         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 307         for (FlowEntryInstall entry : toInstallList) {
 308             // Conflict Check: Verify new entry would not overwrite existing
 309             // ones
 310             if (this.installedSwView.containsKey(entry)) {
 311                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
 312                 log.trace("Aborting to install {}", entry);
 313                 continue;
 314             }
 315             toInstallSafe.add(entry);
 316         }
 317
 318         // Declare failure if all the container flow merged entries clash with
 319         // existing entries
 320         if (toInstallSafe.size() == 0) {
 321             String msg = "A flow with same match and priority exists on the target node";
 322             String logMsg = msg + ": {}";
 323             log.warn(logMsg, flowEntry);
 324             return new Status(StatusCode.CONFLICT, msg);
 325         }
 326
 327         // Try to install an entry at the time
 328         Status error = new Status(null, null);
 329         Status succeded = null;
 330         boolean oneSucceded = false;
 331         for (FlowEntryInstall installEntry : toInstallSafe) {
 332
 333             // Install and update database
 334             Status ret = addEntryInternal(installEntry, async);
 335
 336             if (ret.isSuccess()) {
 337                 oneSucceded = true;
 338                 /*
 339                  * The first successful status response will be returned For the
 340                  * asynchronous call, we can discard the container flow
 341                  * complication for now and assume we will always deal with one
 342                  * flow only per request
 343                  */
 344                 succeded = ret;
 345             } else {
 346                 error = ret;
 347                 log.trace("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
 348             }
 349         }
 350
 351         return (oneSucceded) ? succeded : error;
 352     }
 353
 354     /**
 355      * Given a flow entry and the list of container flows, it returns the list
 356      * of container flow merged flow entries good to be installed on this
 357      * container. If the list of container flows is null or empty, the install
 358      * entry list will contain only one entry, the original flow entry. If the
 359      * flow entry is congruent with all the N container flows, then the output
 360      * install entry list will contain N entries. If the output list is empty,
 361      * it means the passed flow entry conflicts with all the container flows.
 362      *
 363      * @param cFlowList
 364      *            The list of container flows
 365      * @return the list of container flow merged entries good to be installed on
 366      *         this container
 367      */
 368     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
 369         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
 370
 371         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
 372             // No container flows => entry good to be installed unchanged
 373             toInstallList.add(new FlowEntryInstall(request.clone(), null));
 374         } else {
 375             // Create the list of entries to be installed. If the flow entry is
 376             // not congruent with any container flow, no install entries will be
 377             // created
 378             for (ContainerFlow cFlow : container.getContainerFlows()) {
 379                 if (cFlow.allowsFlow(request.getFlow())) {
 380                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
 381                 }
 382             }
 383         }
 384         return toInstallList;
 385     }
 386
 387     /**
 388      * Modify a flow entry with a new one It runs various validity check and
 389      * derive the final container flows merged flow entries to work with
 390      *
 391      * @param currentFlowEntry
 392      * @param newFlowEntry
 393      * @param async
 394      *            the flag indicating if this is a asynchronous request
 395      * @return the status of this request. In case of asynchronous call, it will
 396      *         contain the unique id assigned to this request
 397      */
 398     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
 399         Status retExt;
 400
 401         // Sanity checks
 402         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
 403                 || newFlowEntry.getNode() == null || newFlowEntry.getFlow() == null) {
 404             String msg = "Modify: " + INVALID_FLOW_ENTRY;
 405             String logMsg = msg + ": {} or {}";
 406             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 407             return new Status(StatusCode.NOTACCEPTABLE, msg);
 408         }
 409         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
 410                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
 411             String msg = "Modify: Incompatible Flow Entries";
 412             String logMsg = msg + ": {} and {}";
 413             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 414             return new Status(StatusCode.NOTACCEPTABLE, msg);
 415         }
 416
 417         // Equality Check
 418         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
 419             String msg = "Modify skipped as flows are the same";
 420             String logMsg = msg + ": {} and {}";
 421             log.debug(logMsg, currentFlowEntry, newFlowEntry);
 422             return new Status(StatusCode.SUCCESS, msg);
 423         }
 424
 425         /*
 426          * Conflict Check: Verify the new entry would not conflict with an
 427          * existing one. This is a loose check on the previous original flow
 428          * entry requests. No check on the container flow merged flow entries
 429          * (if any) yet
 430          */
 431         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
 432         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
 433             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
 434             String logMsg = msg + ": {}";
 435             log.warn(logMsg, currentFlowEntry);
 436             return new Status(StatusCode.CONFLICT, msg);
 437         }
 438
 439         // Derive the installed and toInstall entries
 440         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
 441                 container.getContainerFlows());
 442         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
 443
 444         if (toInstallList.isEmpty()) {
 445             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
 446             String logMsg = msg + ": {}";
 447             log.warn(logMsg, newFlowEntry);
 448             log.warn(msg);
 449             return new Status(StatusCode.CONFLICT, msg);
 450         }
 451
 452         /*
 453          * If the two list sizes differ, it means the new flow entry does not
 454          * satisfy the same number of container flows the current entry does.
 455          * This is only possible when the new entry and current entry have
 456          * different match. In this scenario the modification would ultimately
 457          * be handled as a remove and add operations in the protocol plugin.
 458          *
 459          * Also, if any of the new flow entries would clash with an existing
 460          * one, we cannot proceed with the modify operation, because it would
 461          * fail for some entries and leave stale entries on the network node.
 462          * Modify path can be taken only if it can be performed completely, for
 463          * all entries.
 464          *
 465          * So, for the above two cases, to simplify, let's decouple the modify
 466          * in: 1) remove current entries 2) install new entries
 467          */
 468         Status succeeded = null;
 469         boolean decouple = false;
 470         if (installedList.size() != toInstallList.size()) {
 471             log.trace("Modify: New flow entry does not satisfy the same "
 472                     + "number of container flows as the original entry does");
 473             decouple = true;
 474         }
 475         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 476         for (FlowEntryInstall installEntry : toInstallList) {
 477             /*
 478              * Conflict Check: Verify the new entry would not overwrite another
 479              * existing one
 480              */
 481             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
 482             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
 483                 log.trace("Modify: new container flow merged flow entry clashes with existing flow");
 484                 decouple = true;
 485             } else {
 486                 toInstallSafe.add(installEntry);
 487             }
 488         }
 489
 490         if (decouple) {
 491             // Remove current entries
 492             for (FlowEntryInstall currEntry : installedList) {
 493                 this.removeEntryInternal(currEntry, async);
 494             }
 495             // Install new entries
 496             for (FlowEntryInstall newEntry : toInstallSafe) {
 497                 succeeded = this.addEntryInternal(newEntry, async);
 498             }
 499         } else {
 500             /*
 501              * The two list have the same size and the entries to install do not
 502              * clash with any existing flow on the network node. We assume here
 503              * (and might be wrong) that the same container flows that were
 504              * satisfied by the current entries are the same that are satisfied
 505              * by the new entries. Let's take the risk for now.
 506              *
 507              * Note: modification has to be complete. If any entry modification
 508              * fails, we need to stop, restore the already modified entries, and
 509              * declare failure.
 510              */
 511             Status retModify = null;
 512             int i = 0;
 513             int size = toInstallList.size();
 514             while (i < size) {
 515                 // Modify and update database
 516                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
 517                 if (retModify.isSuccess()) {
 518                     i++;
 519                 } else {
 520                     break;
 521                 }
 522             }
 523             // Check if uncompleted modify
 524             if (i < size) {
 525                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
 526                 // Restore original entries
 527                 int j = 0;
 528                 while (j < i) {
 529                     log.info("Attempting to restore initial entries");
 530                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
 531                     if (retExt.isSuccess()) {
 532                         j++;
 533                     } else {
 534                         break;
 535                     }
 536                 }
 537                 // Fatal error, recovery failed
 538                 if (j < i) {
 539                     String msg = "Flow recovery failed ! Unrecoverable Error";
 540                     log.error(msg);
 541                     return new Status(StatusCode.INTERNALERROR, msg);
 542                 }
 543             }
 544             succeeded = retModify;
 545         }
 546         /*
 547          * The first successful status response will be returned. For the
 548          * asynchronous call, we can discard the container flow complication for
 549          * now and assume we will always deal with one flow only per request
 550          */
 551         return succeeded;
 552     }
 553
 554     /**
 555      * This is the function that modifies the final container flows merged
 556      * entries on the network node and update the database. It expects that all
 557      * the validity checks are passed.
 558      * This function is supposed to be called only on the controller on which
 559      * the IFRM call is executed.
 560      *
 561      * @param currentEntries
 562      * @param newEntries
 563      * @param async
 564      *            the flag indicating if this is a asynchronous request
 565      * @return the status of this request. In case of asynchronous call, it will
 566      *         contain the unique id assigned to this request
 567      */
 568     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 569         Status status = new Status(StatusCode.UNDEFINED);
 570         FlowEntryDistributionOrderFutureTask futureStatus =
 571                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
 572         if (futureStatus != null) {
 573             try {
 574                 status = futureStatus.get();
 575                 if (status.getCode()
 576                         .equals(StatusCode.TIMEOUT)) {
 577                     // A timeout happened, lets cleanup the workMonitor
 578                     workMonitor.remove(futureStatus.getOrder());
 579                 }
 580             } catch (InterruptedException e) {
 581                 log.error("", e);
 582             } catch (ExecutionException e) {
 583                 log.error("", e);
 584             }
 585         } else {
 586             // Modify the flow on the network node
 587             status = modifyEntryInHw(currentEntries, newEntries, async);
 588         }
 589
 590         if (!status.isSuccess()) {
 591             log.trace("{} SDN Plugin failed to program the flow: {}. The failure is: {}",
 592                     (futureStatus != null) ? "Remote" : "Local", newEntries.getInstall(), status.getDescription());
 593             return status;
 594         }
 595
 596         log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
 597
 598         // Update DB
 599         newEntries.setRequestId(status.getRequestId());
 600         updateSwViews(currentEntries, false);
 601         updateSwViews(newEntries, true);
 602
 603         return status;
 604     }
 605
 606     private Status modifyEntryInHw(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 607         return async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall().getFlow(),
 608                 newEntries.getInstall().getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries
 609                 .getInstall().getFlow(), newEntries.getInstall().getFlow());
 610     }
 611
 612     /**
 613      * Remove a flow entry. If the entry is not present in the software view
 614      * (entry or node not present), it return successfully
 615      *
 616      * @param flowEntry
 617      *            the flow entry to remove
 618      * @param async
 619      *            the flag indicating if this is a asynchronous request
 620      * @return the status of this request. In case of asynchronous call, it will
 621      *         contain the unique id assigned to this request
 622      */
 623     private Status removeEntry(FlowEntry flowEntry, boolean async) {
 624         Status error = new Status(null, null);
 625
 626         // Sanity Check
 627         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 628             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 629             log.warn(logMsg, flowEntry);
 630             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 631         }
 632
 633         // Derive the container flows merged installed entries
 634         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 635
 636         Status succeeded = null;
 637         boolean atLeastOneRemoved = false;
 638         for (FlowEntryInstall entry : installedList) {
 639             if (!installedSwView.containsKey(entry)) {
 640                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
 641                 log.debug(logMsg, flowEntry);
 642                 if (installedList.size() == 1) {
 643                     // If we had only one entry to remove, we are done
 644                     return new Status(StatusCode.SUCCESS);
 645                 } else {
 646                     continue;
 647                 }
 648             }
 649
 650             // Remove and update DB
 651             Status ret = removeEntryInternal(entry, async);
 652
 653             if (!ret.isSuccess()) {
 654                 error = ret;
 655                 log.trace("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
 656                 if (installedList.size() == 1) {
 657                     // If we had only one entry to remove, this is fatal failure
 658                     return error;
 659                 }
 660             } else {
 661                 succeeded = ret;
 662                 atLeastOneRemoved = true;
 663             }
 664         }
 665
 666         /*
 667          * No worries if full removal failed. Consistency checker will take care
 668          * of removing the stale entries later, or adjusting the software
 669          * database if not in sync with hardware
 670          */
 671         return (atLeastOneRemoved) ? succeeded : error;
 672     }
 673
 674     /**
 675      * This is the function that removes the final container flows merged entry
 676      * from the network node and update the database. It expects that all the
 677      * validity checks are passed
 678      * This function is supposed to be called only on the controller on which
 679      * the IFRM call is executed.
 680      *
 681      * @param entry
 682      *            the flow entry to remove
 683      * @param async
 684      *            the flag indicating if this is a asynchronous request
 685      * @return the status of this request. In case of asynchronous call, it will
 686      *         contain the unique id assigned to this request
 687      */
 688     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
 689         Status status = new Status(StatusCode.UNDEFINED);
 690         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
 691         if (futureStatus != null) {
 692             try {
 693                 status = futureStatus.get();
 694                 if (status.getCode().equals(StatusCode.TIMEOUT)) {
 695                     // A timeout happened, lets cleanup the workMonitor
 696                     workMonitor.remove(futureStatus.getOrder());
 697                 }
 698             } catch (InterruptedException e) {
 699                 log.error("", e);
 700             } catch (ExecutionException e) {
 701                 log.error("", e);
 702             }
 703         } else {
 704             // Mark the entry to be deleted (for CC just in case we fail)
 705             entry.toBeDeleted();
 706
 707             // Remove from node
 708             status = removeEntryInHw(entry, async);
 709         }
 710
 711         if (!status.isSuccess()) {
 712             log.trace("{} SDN Plugin failed to remove the flow: {}. The failure is: {}",
 713                     (futureStatus != null) ? "Remote" : "Local", entry.getInstall(), status.getDescription());
 714             return status;
 715         }
 716
 717         log.trace("Removed  {}", entry.getInstall());
 718
 719         // Update DB
 720         updateSwViews(entry, false);
 721
 722         return status;
 723     }
 724
 725     private Status removeEntryInHw(FlowEntryInstall entry, boolean async) {
 726         return async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall().getFlow()) : programmer
 727                 .removeFlow(entry.getNode(), entry.getInstall().getFlow());
 728     }
 729
 730     /**
 731      * This is the function that installs the final container flow merged entry
 732      * on the network node and updates the database. It expects that all the
 733      * validity and conflict checks are passed. That means it does not check
 734      * whether this flow would conflict or overwrite an existing one.
 735      * This function is supposed to be called only on the controller on which
 736      * the IFRM call is executed.
 737      *
 738      * @param entry
 739      *            the flow entry to install
 740      * @param async
 741      *            the flag indicating if this is a asynchronous request
 742      * @return the status of this request. In case of asynchronous call, it will
 743      *         contain the unique id assigned to this request
 744      */
 745     private Status addEntryInternal(FlowEntryInstall entry, boolean async) {
 746         Status status = new Status(StatusCode.UNDEFINED);
 747         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
 748         if (futureStatus != null) {
 749             try {
 750                 status = futureStatus.get();
 751                 if (status.getCode().equals(StatusCode.TIMEOUT)) {
 752                     // A timeout happened, lets cleanup the workMonitor
 753                     workMonitor.remove(futureStatus.getOrder());
 754                 }
 755             } catch (InterruptedException e) {
 756                 log.error("", e);
 757             } catch (ExecutionException e) {
 758                 log.error("", e);
 759             }
 760         } else {
 761             status = addEntryInHw(entry, async);
 762         }
 763
 764         if (!status.isSuccess()) {
 765             log.trace("{} SDN Plugin failed to program the flow: {}. The failure is: {}",
 766                     (futureStatus != null) ? "Remote" : "Local", entry.getInstall(), status.getDescription());
 767             return status;
 768         }
 769
 770         log.trace("Added    {}", entry.getInstall());
 771
 772         // Update DB
 773         entry.setRequestId(status.getRequestId());
 774         updateSwViews(entry, true);
 775
 776         return status;
 777     }
 778
 779     private Status addEntryInHw(FlowEntryInstall entry, boolean async) {
 780         // Install the flow on the network node
 781         return async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall().getFlow()) : programmer.addFlow(
 782                 entry.getNode(), entry.getInstall().getFlow());
 783     }
 784
 785     /**
 786      * Returns true if the flow conflicts with all the container's flows. This
 787      * means that if the function returns true, the passed flow entry is
 788      * congruent with at least one container flow, hence it is good to be
 789      * installed on this container.
 790      *
 791      * @param flowEntry
 792      * @return true if flow conflicts with all the container flows, false
 793      *         otherwise
 794      */
 795     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
 796         List<ContainerFlow> cFlowList = container.getContainerFlows();
 797
 798         // Validity check and avoid unnecessary computation
 799         // Also takes care of default container where no container flows are
 800         // present
 801         if (cFlowList == null || cFlowList.isEmpty()) {
 802             return false;
 803         }
 804
 805         for (ContainerFlow cFlow : cFlowList) {
 806             if (cFlow.allowsFlow(flowEntry.getFlow())) {
 807                 // Entry is allowed by at least one container flow: good to go
 808                 return false;
 809             }
 810         }
 811         return true;
 812     }
 813
 814     private ConcurrentMap.Entry<Integer, FlowConfig> getStaticFlowEntry(String name, Node node) {
 815         for (ConcurrentMap.Entry<Integer, FlowConfig> flowEntry : staticFlows.entrySet()) {
 816             FlowConfig flowConfig = flowEntry.getValue();
 817             if (flowConfig.isByNameAndNodeIdEqual(name, node)) {
 818                 return flowEntry;
 819             }
 820         }
 821         return null;
 822     }
 823
 824     private void updateIndexDatabase(FlowEntryInstall entry, boolean add) {
 825         // Update node indexed flow database
 826         updateNodeFlowsDB(entry, add);
 827
 828         // Update group indexed flow database
 829         updateGroupFlowsDB(entry, add);
 830     }
 831
 832     /*
 833      * Update the node mapped flows database
 834      */
 835     private void updateSwViews(FlowEntryInstall flowEntries, boolean add) {
 836         if (add) {
 837             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
 838             installedSwView.put(flowEntries, flowEntries);
 839         } else {
 840             originalSwView.remove(flowEntries.getOriginal());
 841             installedSwView.remove(flowEntries);
 842         }
 843     }
 844
 845     /*
 846      * Update the node mapped flows database
 847      */
 848     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 849         Node node = flowEntries.getNode();
 850
 851         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
 852         if (nodeIndeces == null) {
 853             if (!add) {
 854                 return;
 855             } else {
 856                 nodeIndeces = new ArrayList<FlowEntryInstall>();
 857             }
 858         }
 859
 860         if (add) {
 861             // there may be an already existing entry.
 862             // remove it before adding the new one.
 863             // This is necessary since we have observed that in some cases
 864             // Infinispan does aggregation for operations (eg:- remove and then put a different value)
 865             // related to the same key within the same transaction.
 866             // Need this defensive code as the new FlowEntryInstall may be different
 867             // than the old one even though the equals method returns true. This is because
 868             // the equals method does not take into account the action list.
 869             if(nodeIndeces.contains(flowEntries)) {
 870                 nodeIndeces.remove(flowEntries);
 871             }
 872             nodeIndeces.add(flowEntries);
 873         } else {
 874             nodeIndeces.remove(flowEntries);
 875         }
 876
 877         // Update cache across cluster
 878         if (nodeIndeces.isEmpty()) {
 879             this.nodeFlows.remove(node);
 880         } else {
 881             this.nodeFlows.put(node, nodeIndeces);
 882         }
 883     }
 884
 885     /*
 886      * Update the group name mapped flows database
 887      */
 888     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 889         String groupName = flowEntries.getGroupName();
 890
 891         // Flow may not be part of a group
 892         if (groupName == null) {
 893             return;
 894         }
 895
 896         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
 897         if (indices == null) {
 898             if (!add) {
 899                 return;
 900             } else {
 901                 indices = new ArrayList<FlowEntryInstall>();
 902             }
 903         }
 904
 905         if (add) {
 906             // same comments in the similar code section in
 907             // updateNodeFlowsDB method apply here too
 908             if(indices.contains(flowEntries)) {
 909                 indices.remove(flowEntries);
 910             }
 911             indices.add(flowEntries);
 912         } else {
 913             indices.remove(flowEntries);
 914         }
 915
 916         // Update cache across cluster
 917         if (indices.isEmpty()) {
 918             this.groupFlows.remove(groupName);
 919         } else {
 920             this.groupFlows.put(groupName, indices);
 921         }
 922     }
 923
 924     /**
 925      * Remove a flow entry that has been added previously First checks if the
 926      * entry is effectively present in the local database
 927      */
 928     @SuppressWarnings("unused")
 929     private Status removeEntry(Node node, String flowName) {
 930         FlowEntryInstall target = null;
 931
 932         // Find in database
 933         for (FlowEntryInstall entry : installedSwView.values()) {
 934             if (entry.equalsByNodeAndName(node, flowName)) {
 935                 target = entry;
 936                 break;
 937             }
 938         }
 939
 940         // If it is not there, stop any further processing
 941         if (target == null) {
 942             return new Status(StatusCode.SUCCESS, "Entry is not present");
 943         }
 944
 945         // Remove from node
 946         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
 947
 948         // Update DB
 949         if (status.isSuccess()) {
 950             updateSwViews(target, false);
 951         } else {
 952             // log the error
 953             log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
 954                     status.getDescription());
 955         }
 956
 957         return status;
 958     }
 959
 960     @Override
 961     public Status installFlowEntry(FlowEntry flowEntry) {
 962         Status status;
 963         if (isContainerModeAllowed(flowEntry)) {
 964             status = addEntry(flowEntry, false);
 965         } else {
 966             String msg = "Controller in container mode: Install Refused";
 967             String logMsg = msg + ": {}";
 968             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 969             log.warn(logMsg, flowEntry);
 970         }
 971         return status;
 972     }
 973
 974     @Override
 975     public Status installFlowEntryAsync(FlowEntry flowEntry) {
 976         Status status;
 977         if (isContainerModeAllowed(flowEntry)) {
 978             status = addEntry(flowEntry, true);
 979         } else {
 980             String msg = "Controller in container mode: Install Refused";
 981             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 982             log.warn(msg);
 983         }
 984         return status;
 985     }
 986
 987     @Override
 988     public Status uninstallFlowEntry(FlowEntry flowEntry) {
 989         Status status;
 990         if (isContainerModeAllowed(flowEntry)) {
 991             status = removeEntry(flowEntry, false);
 992         } else {
 993             String msg = "Controller in container mode: Uninstall Refused";
 994             String logMsg = msg + ": {}";
 995             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 996             log.warn(logMsg, flowEntry);
 997         }
 998         return status;
 999     }
1000
1001     @Override
1002     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
1003         Status status;
1004         if (isContainerModeAllowed(flowEntry)) {
1005             status = removeEntry(flowEntry, true);
1006         } else {
1007             String msg = "Controller in container mode: Uninstall Refused";
1008             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1009             log.warn(msg);
1010         }
1011         return status;
1012     }
1013
1014     @Override
1015     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1016         Status status = null;
1017         if (isContainerModeAllowed(currentFlowEntry)) {
1018             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1019         } else {
1020             String msg = "Controller in container mode: Modify Refused";
1021             String logMsg = msg + ": {}";
1022             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1023             log.warn(logMsg, newFlowEntry);
1024         }
1025         return status;
1026     }
1027
1028     @Override
1029     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1030         Status status = null;
1031         if (isContainerModeAllowed(currentFlowEntry)) {
1032             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
1033         } else {
1034             String msg = "Controller in container mode: Modify Refused";
1035             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1036             log.warn(msg);
1037         }
1038         return status;
1039     }
1040
1041     /**
1042      * Returns whether the specified flow entry is allowed to be
1043      * installed/removed/modified based on the current container mode status.
1044      * This call always returns true in the container instance of forwarding
1045      * rules manager. It is meant for the global instance only (default
1046      * container) of forwarding rules manager. Idea is that for assuring
1047      * container isolation of traffic, flow installation in default container is
1048      * blocked when in container mode (containers are present). The only flows
1049      * that are allowed in container mode in the default container are the
1050      * proactive flows, the ones automatically installed on the network node
1051      * which forwarding mode has been configured to "proactive". These flows are
1052      * needed by controller to discover the nodes topology and to discover the
1053      * attached hosts for some SDN switches.
1054      *
1055      * @param flowEntry
1056      *            The flow entry to be installed/removed/modified
1057      * @return true if not in container mode or if flowEntry is internally
1058      *         generated
1059      */
1060     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1061         return (!inContainerMode) ? true : flowEntry.isInternal();
1062     }
1063
1064     @Override
1065     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1066         /*
1067          * Run a check on the original entries to decide whether to go with a
1068          * add or modify method. A loose check means only check against the
1069          * original flow entry requests and not against the installed flow
1070          * entries which are the result of the original entry merged with the
1071          * container flow(s) (if any). The modifyFlowEntry method in presence of
1072          * conflicts with the Container flows (if any) would revert back to a
1073          * delete + add pattern
1074          */
1075         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1076
1077         if (currentFlowEntry != null) {
1078             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1079         } else {
1080             return installFlowEntry(newFlowEntry);
1081         }
1082     }
1083
1084     @Override
1085     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1086         /*
1087          * Run a check on the original entries to decide whether to go with a
1088          * add or modify method. A loose check means only check against the
1089          * original flow entry requests and not against the installed flow
1090          * entries which are the result of the original entry merged with the
1091          * container flow(s) (if any). The modifyFlowEntry method in presence of
1092          * conflicts with the Container flows (if any) would revert back to a
1093          * delete + add pattern
1094          */
1095         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1096
1097         if (currentFlowEntry != null) {
1098             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1099         } else {
1100             return installFlowEntryAsync(newFlowEntry);
1101         }
1102     }
1103
1104     @Override
1105     public Status uninstallFlowEntryGroup(String groupName) {
1106         if (groupName == null || groupName.isEmpty()) {
1107             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1108         }
1109         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1110             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1111         }
1112         if (inContainerMode) {
1113             String msg = "Controller in container mode: Group Uninstall Refused";
1114             String logMsg = msg + ": {}";
1115             log.warn(logMsg, groupName);
1116             return new Status(StatusCode.NOTACCEPTABLE, msg);
1117         }
1118         int toBeRemoved = 0;
1119         String error = "";
1120         if (groupFlows.containsKey(groupName)) {
1121             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1122             toBeRemoved = list.size();
1123             for (FlowEntryInstall entry : list) {
1124                 Status status = this.removeEntry(entry.getOriginal(), false);
1125                 if (status.isSuccess()) {
1126                     toBeRemoved -= 1;
1127                 } else {
1128                     error = status.getDescription();
1129                 }
1130             }
1131         }
1132         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1133                 "Not all the flows were removed: " + error);
1134     }
1135
1136     @Override
1137     public Status uninstallFlowEntryGroupAsync(String groupName) {
1138         if (groupName == null || groupName.isEmpty()) {
1139             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1140         }
1141         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1142             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1143         }
1144         if (inContainerMode) {
1145             String msg = "Controller in container mode: Group Uninstall Refused";
1146             String logMsg = msg + ": {}";
1147             log.warn(logMsg, groupName);
1148             return new Status(StatusCode.NOTACCEPTABLE, msg);
1149         }
1150         if (groupFlows.containsKey(groupName)) {
1151             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1152             for (FlowEntryInstall entry : list) {
1153                 this.removeEntry(entry.getOriginal(), true);
1154             }
1155         }
1156         return new Status(StatusCode.SUCCESS);
1157     }
1158
1159     @Override
1160     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1161         return entryConflictsWithContainerFlows(flowEntry);
1162     }
1163
1164     /**
1165      * Updates all installed flows because the container flow got updated This
1166      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1167      * Reinstall of all flows This is needed because a new container flows
1168      * merged flow may conflict with an existing old container flows merged flow
1169      * on the network node
1170      */
1171     protected void updateFlowsContainerFlow() {
1172         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1173         // First remove all installed entries
1174         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1175             FlowEntryInstall current = entry.getValue();
1176             // Store the original entry
1177             toReInstall.add(current.getOriginal());
1178             // Remove the old couples. No validity checks to be run, use the
1179             // internal remove
1180             this.removeEntryInternal(current, false);
1181         }
1182         // Then reinstall the original entries
1183         for (FlowEntry entry : toReInstall) {
1184             // Reinstall the original flow entries, via the regular path: new
1185             // cFlow merge + validations
1186             this.installFlowEntry(entry);
1187         }
1188     }
1189
1190     private void nonClusterObjectCreate() {
1191         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1192         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1193         TSPolicies = new ConcurrentHashMap<String, Object>();
1194         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1195         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1196         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1197         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1198         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1199     }
1200
1201     @Override
1202     public void setTSPolicyData(String policyname, Object o, boolean add) {
1203
1204         if (add) {
1205             /* Check if this policy already exists */
1206             if (!(TSPolicies.containsKey(policyname))) {
1207                 TSPolicies.put(policyname, o);
1208             }
1209         } else {
1210             TSPolicies.remove(policyname);
1211         }
1212         if (frmAware != null) {
1213             synchronized (frmAware) {
1214                 for (IForwardingRulesManagerAware frma : frmAware) {
1215                     try {
1216                         frma.policyUpdate(policyname, add);
1217                     } catch (Exception e) {
1218                         log.warn("Exception on callback", e);
1219                     }
1220                 }
1221             }
1222         }
1223     }
1224
1225     @Override
1226     public Map<String, Object> getTSPolicyData() {
1227         return TSPolicies;
1228     }
1229
1230     @Override
1231     public Object getTSPolicyData(String policyName) {
1232         if (TSPolicies.containsKey(policyName)) {
1233             return TSPolicies.get(policyName);
1234         } else {
1235             return null;
1236         }
1237     }
1238
1239     @Override
1240     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1241         List<FlowEntry> list = new ArrayList<FlowEntry>();
1242         if (policyName != null && !policyName.trim().isEmpty()) {
1243             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1244                 if (policyName.equals(entry.getKey().getGroupName())) {
1245                     list.add(entry.getValue().clone());
1246                 }
1247             }
1248         }
1249         return list;
1250     }
1251
1252     @Override
1253     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1254         List<FlowEntry> list = new ArrayList<FlowEntry>();
1255         if (policyName != null && !policyName.trim().isEmpty()) {
1256             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1257                 if (policyName.equals(entry.getKey().getGroupName())) {
1258                     list.add(entry.getValue().getInstall().clone());
1259                 }
1260             }
1261         }
1262         return list;
1263     }
1264
1265     @Override
1266     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1267
1268         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1269             if (flow.getFlowName().equals(flowName)) {
1270                 FlowEntry currentFlowEntry = flow.getOriginal();
1271                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1272                 for (NodeConnector dstPort : portList) {
1273                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1274                 }
1275                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1276                 if (error.isSuccess()) {
1277                     log.trace("Ports {} added to FlowEntry {}", portList, flowName);
1278                 } else {
1279                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1280                             currentFlowEntry.toString(), error.getDescription());
1281                 }
1282                 return;
1283             }
1284         }
1285         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1286     }
1287
1288     @Override
1289     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1290         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1291             FlowEntryInstall flow = this.installedSwView.get(index);
1292             if (flow.getFlowName().equals(flowName)) {
1293                 FlowEntry currentFlowEntry = flow.getOriginal();
1294                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1295                 for (NodeConnector dstPort : portList) {
1296                     Action action = new Output(dstPort);
1297                     newFlowEntry.getFlow().removeAction(action);
1298                 }
1299                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1300                 if (status.isSuccess()) {
1301                     log.trace("Ports {} removed from FlowEntry {}", portList, flowName);
1302                 } else {
1303                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1304                             currentFlowEntry.toString(), status.getDescription());
1305                 }
1306                 return;
1307             }
1308         }
1309         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1310     }
1311
1312     /*
1313      * This function assumes the target flow has only one output port
1314      */
1315     @Override
1316     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1317         FlowEntry currentFlowEntry = null;
1318         FlowEntry newFlowEntry = null;
1319
1320         // Find the flow
1321         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1322             FlowEntryInstall flow = this.installedSwView.get(index);
1323             if (flow.getFlowName().equals(flowName)) {
1324                 currentFlowEntry = flow.getOriginal();
1325                 break;
1326             }
1327         }
1328         if (currentFlowEntry == null) {
1329             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1330             return;
1331         }
1332
1333         // Create a flow copy with the new output port
1334         newFlowEntry = currentFlowEntry.clone();
1335         Action target = null;
1336         for (Action action : newFlowEntry.getFlow().getActions()) {
1337             if (action.getType() == ActionType.OUTPUT) {
1338                 target = action;
1339                 break;
1340             }
1341         }
1342         newFlowEntry.getFlow().removeAction(target);
1343         newFlowEntry.getFlow().addAction(new Output(outPort));
1344
1345         // Modify on network node
1346         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1347
1348         if (status.isSuccess()) {
1349             log.trace("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1350         } else {
1351             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1352                     status.getDescription());
1353         }
1354         return;
1355     }
1356
1357     @Override
1358     public NodeConnector getOutputPort(Node node, String flowName) {
1359         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1360             FlowEntryInstall flow = this.installedSwView.get(index);
1361             if (flow.getFlowName().equals(flowName)) {
1362                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1363                     if (action.getType() == ActionType.OUTPUT) {
1364                         return ((Output) action).getPort();
1365                     }
1366                 }
1367             }
1368         }
1369         return null;
1370     }
1371
1372     private void cacheStartup() {
1373         allocateCaches();
1374         retrieveCaches();
1375     }
1376
1377     private void allocateCaches() {
1378         if (this.clusterContainerService == null) {
1379             log.warn("Un-initialized clusterContainerService, can't create cache");
1380             return;
1381         }
1382
1383         log.debug("Allocating caches for Container {}", container.getName());
1384
1385         try {
1386             clusterContainerService.createCache(ORIGINAL_SW_VIEW_CACHE,
1387                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1388
1389             clusterContainerService.createCache(INSTALLED_SW_VIEW_CACHE,
1390                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1391
1392             clusterContainerService.createCache("frm.inactiveFlows",
1393                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1394
1395             clusterContainerService.createCache("frm.staticFlows",
1396                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1397
1398             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1399                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1400
1401             clusterContainerService.createCache("frm.portGroupConfigs",
1402                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1403
1404             clusterContainerService.createCache("frm.portGroupData",
1405                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1406
1407             clusterContainerService.createCache("frm.TSPolicies",
1408                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1409
1410             clusterContainerService.createCache(WORK_STATUS_CACHE,
1411                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1412
1413             clusterContainerService.createCache(WORK_ORDER_CACHE,
1414                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1415
1416         } catch (CacheConfigException cce) {
1417             log.error("CacheConfigException");
1418         } catch (CacheExistException cce) {
1419             log.error("CacheExistException");
1420         }
1421     }
1422
1423     @SuppressWarnings({ "unchecked" })
1424     private void retrieveCaches() {
1425         ConcurrentMap<?, ?> map;
1426
1427         if (this.clusterContainerService == null) {
1428             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1429             nonClusterObjectCreate();
1430             return;
1431         }
1432
1433         log.debug("Retrieving Caches for Container {}", container.getName());
1434
1435         map = clusterContainerService.getCache(ORIGINAL_SW_VIEW_CACHE);
1436         if (map != null) {
1437             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1438         } else {
1439             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1440         }
1441
1442         map = clusterContainerService.getCache(INSTALLED_SW_VIEW_CACHE);
1443         if (map != null) {
1444             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1445         } else {
1446             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1447         }
1448
1449         map = clusterContainerService.getCache("frm.inactiveFlows");
1450         if (map != null) {
1451             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1452         } else {
1453             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1454         }
1455
1456         map = clusterContainerService.getCache("frm.staticFlows");
1457         if (map != null) {
1458             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1459         } else {
1460             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1461         }
1462
1463         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1464         if (map != null) {
1465             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1466         } else {
1467             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1468         }
1469
1470         map = clusterContainerService.getCache("frm.portGroupConfigs");
1471         if (map != null) {
1472             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1473         } else {
1474             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1475         }
1476
1477         map = clusterContainerService.getCache("frm.portGroupData");
1478         if (map != null) {
1479             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1480         } else {
1481             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1482         }
1483
1484         map = clusterContainerService.getCache("frm.TSPolicies");
1485         if (map != null) {
1486             TSPolicies = (ConcurrentMap<String, Object>) map;
1487         } else {
1488             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1489         }
1490
1491         map = clusterContainerService.getCache(WORK_ORDER_CACHE);
1492         if (map != null) {
1493             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1494         } else {
1495             log.error("Retrieval of " + WORK_ORDER_CACHE + " cache failed for Container {}", container.getName());
1496         }
1497
1498         map = clusterContainerService.getCache(WORK_STATUS_CACHE);
1499         if (map != null) {
1500             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1501         } else {
1502             log.error("Retrieval of " + WORK_STATUS_CACHE + " cache failed for Container {}", container.getName());
1503         }
1504     }
1505
1506     private boolean flowConfigExists(FlowConfig config) {
1507         // Flow name has to be unique on per node id basis
1508         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1509             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1510                 return true;
1511             }
1512         }
1513         return false;
1514     }
1515
1516     @Override
1517     public Status addStaticFlow(FlowConfig config) {
1518         // Configuration object validation
1519         Status status = config.validate(container);
1520         if (!status.isSuccess()) {
1521             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1522             String error = "Invalid Configuration (" + status.getDescription() + ")";
1523             config.setStatus(error);
1524             return new Status(StatusCode.BADREQUEST, error);
1525         }
1526         return addStaticFlowInternal(config, false);
1527     }
1528
1529     /**
1530      * Private method to add a static flow configuration which does not run any
1531      * validation on the passed FlowConfig object. If restore is set to true,
1532      * configuration is stored in configuration database regardless the
1533      * installation on the network node was successful. This is useful at boot
1534      * when static flows are present in startup configuration and are read
1535      * before the switches connects.
1536      *
1537      * @param config
1538      *            The static flow configuration
1539      * @param restore
1540      *            if true, the configuration is stored regardless the
1541      *            installation on the network node was successful
1542      * @return The status of this request
1543      */
1544     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1545         boolean multipleFlowPush = false;
1546         String error;
1547         Status status;
1548         config.setStatus(StatusCode.SUCCESS.toString());
1549
1550         // Presence check
1551         if (flowConfigExists(config)) {
1552             error = "Entry with this name on specified switch already exists";
1553             log.warn("Entry with this name on specified switch already exists: {}", config);
1554             config.setStatus(error);
1555             return new Status(StatusCode.CONFLICT, error);
1556         }
1557
1558         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1559             for (String portGroupName : portGroupConfigs.keySet()) {
1560                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1561                     multipleFlowPush = true;
1562                     break;
1563                 }
1564             }
1565             if (!multipleFlowPush) {
1566                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1567                 error = "Invalid Configuration (Invalid PortGroup Name)";
1568                 config.setStatus(error);
1569                 return new Status(StatusCode.BADREQUEST, error);
1570             }
1571         }
1572
1573         /*
1574          * If requested program the entry in hardware first before updating the
1575          * StaticFlow DB
1576          */
1577         if (!multipleFlowPush) {
1578             // Program hw
1579             if (config.installInHw()) {
1580                 FlowEntry entry = config.getFlowEntry();
1581                 status = this.installFlowEntry(entry);
1582                 if (!status.isSuccess()) {
1583                     config.setStatus(status.getDescription());
1584                     if (!restore) {
1585                         return status;
1586                     }
1587                 }
1588             }
1589         }
1590
1591         /*
1592          * When the control reaches this point, either of the following
1593          * conditions is true 1. This is a single entry configuration (non
1594          * PortGroup) and the hardware installation is successful 2. This is a
1595          * multiple entry configuration (PortGroup) and hardware installation is
1596          * NOT done directly on this event. 3. The User prefers to retain the
1597          * configuration in Controller and skip hardware installation.
1598          *
1599          * Hence it is safe to update the StaticFlow DB at this point.
1600          *
1601          * Note : For the case of PortGrouping, it is essential to have this DB
1602          * populated before the PortGroupListeners can query for the DB
1603          * triggered using portGroupChanged event...
1604          */
1605         Integer ordinal = staticFlowsOrdinal.get(0);
1606         staticFlowsOrdinal.put(0, ++ordinal);
1607         staticFlows.put(ordinal, config);
1608
1609         if (multipleFlowPush) {
1610             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1611             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1612             if (existingData != null) {
1613                 portGroupChanged(pgconfig, existingData, true);
1614             }
1615         }
1616         return new Status(StatusCode.SUCCESS);
1617     }
1618
1619     private void addStaticFlowsToSwitch(Node node) {
1620         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1621             FlowConfig config = entry.getValue();
1622             if (config.isPortGroupEnabled()) {
1623                 continue;
1624             }
1625             if (config.getNode().equals(node)) {
1626                 if (config.installInHw() && !config.getStatus().equals(StatusCode.SUCCESS.toString())) {
1627                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1628                     config.setStatus(status.getDescription());
1629                 }
1630             }
1631         }
1632         // Update cluster cache
1633         refreshClusterStaticFlowsStatus(node);
1634     }
1635
1636     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1637         log.trace("Updating Static Flow configs on node down: {}", node);
1638
1639         List<Integer> toRemove = new ArrayList<Integer>();
1640         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1641
1642             FlowConfig config = entry.getValue();
1643
1644             if (config.isPortGroupEnabled()) {
1645                 continue;
1646             }
1647
1648             if (config.installInHw() && config.getNode().equals(node)) {
1649                 if (config.isInternalFlow()) {
1650                     // Take note of this controller generated static flow
1651                     toRemove.add(entry.getKey());
1652                 } else {
1653                     config.setStatus(NODE_DOWN);
1654                 }
1655             }
1656         }
1657         // Remove controller generated static flows for this node
1658         for (Integer index : toRemove) {
1659             staticFlows.remove(index);
1660         }
1661         // Update cluster cache
1662         refreshClusterStaticFlowsStatus(node);
1663
1664     }
1665
1666     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1667         log.trace("Updating Static Flow configs on container mode change: {}", update);
1668
1669         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1670             FlowConfig config = entry.getValue();
1671             if (config.isPortGroupEnabled()) {
1672                 continue;
1673             }
1674             if (config.installInHw() && !config.isInternalFlow()) {
1675                 switch (update) {
1676                 case ADDED:
1677                     config.setStatus("Removed from node because in container mode");
1678                     break;
1679                 case REMOVED:
1680                     config.setStatus(StatusCode.SUCCESS.toString());
1681                     break;
1682                 default:
1683                 }
1684             }
1685         }
1686         // Update cluster cache
1687         refreshClusterStaticFlowsStatus(null);
1688     }
1689
1690     @Override
1691     public Status removeStaticFlow(FlowConfig config) {
1692         /*
1693          * No config.isInternal() check as NB does not take this path and GUI
1694          * cannot issue a delete on an internal generated flow. We need this
1695          * path to be accessible when switch mode is changed from proactive to
1696          * reactive, so that we can remove the internal generated LLDP and ARP
1697          * punt flows
1698          */
1699
1700         // Look for the target configuration entry
1701         Integer key = 0;
1702         FlowConfig target = null;
1703         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1704             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1705                 key = entry.getKey();
1706                 target = entry.getValue();
1707                 break;
1708             }
1709         }
1710         if (target == null) {
1711             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1712         }
1713
1714         // Program the network node
1715         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1716
1717         // Update configuration database if programming was successful
1718         if (status.isSuccess()) {
1719             staticFlows.remove(key);
1720         }
1721
1722         return status;
1723     }
1724
1725     @Override
1726     public Status removeStaticFlow(String name, Node node) {
1727         // Look for the target configuration entry
1728         Integer key = 0;
1729         FlowConfig target = null;
1730         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1731             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1732                 key = mapEntry.getKey();
1733                 target = mapEntry.getValue();
1734                 break;
1735             }
1736         }
1737         if (target == null) {
1738             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1739         }
1740
1741         // Validity check for api3 entry point
1742         if (target.isInternalFlow()) {
1743             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1744             String logMsg = msg + ": {}";
1745             log.warn(logMsg, name);
1746             return new Status(StatusCode.NOTACCEPTABLE, msg);
1747         }
1748
1749         if (target.isPortGroupEnabled()) {
1750             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1751             String logMsg = msg + ": {}";
1752             log.warn(logMsg, name);
1753             return new Status(StatusCode.NOTACCEPTABLE, msg);
1754         }
1755
1756         // Program the network node
1757         Status status = this.removeEntry(target.getFlowEntry(), false);
1758
1759         // Update configuration database if programming was successful
1760         if (status.isSuccess()) {
1761             staticFlows.remove(key);
1762         }
1763
1764         return status;
1765     }
1766
1767     @Override
1768     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1769         // Validity check for api3 entry point
1770         if (newFlowConfig.isInternalFlow()) {
1771             String msg = "Invalid operation: Controller generated flow cannot be modified";
1772             String logMsg = msg + ": {}";
1773             log.warn(logMsg, newFlowConfig);
1774             return new Status(StatusCode.NOTACCEPTABLE, msg);
1775         }
1776
1777         // Validity Check
1778         Status status = newFlowConfig.validate(container);
1779         if (!status.isSuccess()) {
1780             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1781             newFlowConfig.setStatus(msg);
1782             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1783             return new Status(StatusCode.BADREQUEST, msg);
1784         }
1785
1786         FlowConfig oldFlowConfig = null;
1787         Integer index = null;
1788         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1789             FlowConfig entry = mapEntry.getValue();
1790             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1791                 oldFlowConfig = entry;
1792                 index = mapEntry.getKey();
1793                 break;
1794             }
1795         }
1796
1797         if (oldFlowConfig == null) {
1798             String msg = "Attempt to modify a non existing static flow";
1799             String logMsg = msg + ": {}";
1800             log.warn(logMsg, newFlowConfig);
1801             return new Status(StatusCode.NOTFOUND, msg);
1802         }
1803
1804         // Do not attempt to reinstall the flow, warn user
1805         if (newFlowConfig.equals(oldFlowConfig)) {
1806             String msg = "No modification detected";
1807             log.trace("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1808             return new Status(StatusCode.SUCCESS, msg);
1809         }
1810
1811         // If flow is installed, program the network node
1812         status = new Status(StatusCode.SUCCESS, "Saved in config");
1813         if (oldFlowConfig.installInHw()) {
1814             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1815         }
1816
1817         // Update configuration database if programming was successful
1818         if (status.isSuccess()) {
1819             newFlowConfig.setStatus(status.getDescription());
1820             staticFlows.put(index, newFlowConfig);
1821         }
1822
1823         return status;
1824     }
1825
1826     @Override
1827     public Status toggleStaticFlowStatus(String name, Node node) {
1828         return toggleStaticFlowStatus(getStaticFlow(name, node));
1829     }
1830
1831     @Override
1832     public Status toggleStaticFlowStatus(FlowConfig config) {
1833         if (config == null) {
1834             String msg = "Invalid request: null flow config";
1835             log.warn(msg);
1836             return new Status(StatusCode.BADREQUEST, msg);
1837         }
1838         // Validity check for api3 entry point
1839         if (config.isInternalFlow()) {
1840             String msg = "Invalid operation: Controller generated flow cannot be modified";
1841             String logMsg = msg + ": {}";
1842             log.warn(logMsg, config);
1843             return new Status(StatusCode.NOTACCEPTABLE, msg);
1844         }
1845
1846         // Find the config entry
1847         Integer key = 0;
1848         FlowConfig target = null;
1849         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1850             FlowConfig conf = entry.getValue();
1851             if (conf.isByNameAndNodeIdEqual(config)) {
1852                 key = entry.getKey();
1853                 target = conf;
1854                 break;
1855             }
1856         }
1857         if (target != null) {
1858             Status status = target.validate(container);
1859             if (!status.isSuccess()) {
1860                 log.warn(status.getDescription());
1861                 return status;
1862             }
1863             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1864                                     .installFlowEntry(target.getFlowEntry());
1865             if (status.isSuccess()) {
1866                 // Update Configuration database
1867                 target.setStatus(StatusCode.SUCCESS.toString());
1868                 target.toggleInstallation();
1869                 staticFlows.put(key, target);
1870             }
1871             return status;
1872         }
1873
1874         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1875     }
1876
1877     /**
1878      * Reinsert all static flows entries in the cache to force cache updates in
1879      * the cluster. This is useful when only some parameters were changed in the
1880      * entries, like the status.
1881      *
1882      * @param node
1883      *            The node for which the static flow configurations have to be
1884      *            refreshed. If null, all nodes static flows will be refreshed.
1885      */
1886     private void refreshClusterStaticFlowsStatus(Node node) {
1887         // Refresh cluster cache
1888         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1889             if (node == null || entry.getValue().getNode().equals(node)) {
1890                 staticFlows.put(entry.getKey(), entry.getValue());
1891             }
1892         }
1893     }
1894
1895     /**
1896      * Uninstall all the non-internal Flow Entries present in the software view.
1897      * If requested, a copy of each original flow entry will be stored in the
1898      * inactive list so that it can be re-applied when needed (This is typically
1899      * the case when running in the default container and controller moved to
1900      * container mode) NOTE WELL: The routine as long as does a bulk change will
1901      * operate only on the entries for nodes locally attached so to avoid
1902      * redundant operations initiated by multiple nodes
1903      *
1904      * @param preserveFlowEntries
1905      *            if true, a copy of each original entry is stored in the
1906      *            inactive list
1907      */
1908     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1909         log.trace("Uninstalling all non-internal flows");
1910
1911         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1912
1913         // Store entries / create target list
1914         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1915             FlowEntryInstall flowEntries = mapEntry.getValue();
1916             // Skip internal generated static flows
1917             if (!flowEntries.isInternal()) {
1918                 toRemove.add(flowEntries);
1919                 // Store the original entries if requested
1920                 if (preserveFlowEntries) {
1921                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1922                 }
1923             }
1924         }
1925
1926         // Now remove the entries
1927         for (FlowEntryInstall flowEntryHw : toRemove) {
1928             Node n = flowEntryHw.getNode();
1929             if (n != null && connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
1930                 Status status = this.removeEntryInternal(flowEntryHw, false);
1931                 if (!status.isSuccess()) {
1932                     log.trace("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1933                 }
1934             } else {
1935                 log.debug("Not removing entry {} because not connected locally, the remote guy will do it's job",
1936                         flowEntryHw);
1937             }
1938         }
1939     }
1940
1941     /**
1942      * Re-install all the Flow Entries present in the inactive list The inactive
1943      * list will be empty at the end of this call This function is called on the
1944      * default container instance of FRM only when the last container is deleted
1945      */
1946     private void reinstallAllFlowEntries() {
1947         log.trace("Reinstalling all inactive flows");
1948
1949         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1950             this.addEntry(flowEntry, false);
1951         }
1952
1953         // Empty inactive list in any case
1954         inactiveFlows.clear();
1955     }
1956
1957     @Override
1958     public List<FlowConfig> getStaticFlows() {
1959         return new ArrayList<FlowConfig>(staticFlows.values());
1960     }
1961
1962     @Override
1963     public FlowConfig getStaticFlow(String name, Node node) {
1964         ConcurrentMap.Entry<Integer, FlowConfig> entry = getStaticFlowEntry(name, node);
1965         if(entry != null) {
1966             return entry.getValue();
1967         }
1968         return null;
1969     }
1970
1971     @Override
1972     public List<FlowConfig> getStaticFlows(Node node) {
1973         List<FlowConfig> list = new ArrayList<FlowConfig>();
1974         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1975             if (entry.getValue().onNode(node)) {
1976                 list.add(entry.getValue());
1977             }
1978         }
1979         return list;
1980     }
1981
1982     @Override
1983     public List<String> getStaticFlowNamesForNode(Node node) {
1984         List<String> list = new ArrayList<String>();
1985         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1986             if (entry.getValue().onNode(node)) {
1987                 list.add(entry.getValue().getName());
1988             }
1989         }
1990         return list;
1991     }
1992
1993     @Override
1994     public List<Node> getListNodeWithConfiguredFlows() {
1995         Set<Node> set = new HashSet<Node>();
1996         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1997             set.add(entry.getValue().getNode());
1998         }
1999         return new ArrayList<Node>(set);
2000     }
2001
2002     private void loadFlowConfiguration() {
2003         for (ConfigurationObject conf : configurationService.retrieveConfiguration(this, PORT_GROUP_FILE_NAME)) {
2004             addPortGroupConfig(((PortGroupConfig) conf).getName(), ((PortGroupConfig) conf).getMatchString(), true);
2005         }
2006
2007         for (ConfigurationObject conf : configurationService.retrieveConfiguration(this, STATIC_FLOWS_FILE_NAME)) {
2008             addStaticFlowInternal((FlowConfig) conf, true);
2009         }
2010     }
2011
2012     @Override
2013     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2014         return ois.readObject();
2015     }
2016
2017     @Override
2018     public Status saveConfig() {
2019         return saveConfigInternal();
2020     }
2021
2022     private Status saveConfigInternal() {
2023         List<ConfigurationObject> nonDynamicFlows = new ArrayList<ConfigurationObject>();
2024
2025         for (Integer ordinal : staticFlows.keySet()) {
2026             FlowConfig config = staticFlows.get(ordinal);
2027             // Do not save dynamic and controller generated static flows
2028             if (config.isDynamic() || config.isInternalFlow()) {
2029                 continue;
2030             }
2031             nonDynamicFlows.add(config);
2032         }
2033
2034         configurationService.persistConfiguration(nonDynamicFlows, STATIC_FLOWS_FILE_NAME);
2035         configurationService.persistConfiguration(new ArrayList<ConfigurationObject>(portGroupConfigs.values()),
2036                 PORT_GROUP_FILE_NAME);
2037
2038         return new Status(StatusCode.SUCCESS);
2039     }
2040
2041     @Override
2042     public void subnetNotify(Subnet sub, boolean add) {
2043     }
2044
2045     private boolean programInternalFlow(boolean proactive, FlowConfig fc) {
2046         boolean retVal = true; // program flows unless determined otherwise
2047         if(proactive) {
2048             // if the flow already exists do not program
2049             if(flowConfigExists(fc)) {
2050                 retVal = false;
2051             }
2052         } else {
2053             // if the flow does not exist do not program
2054             if(!flowConfigExists(fc)) {
2055                 retVal = false;
2056             }
2057         }
2058         return retVal;
2059     }
2060
2061     /**
2062      * (non-Javadoc)
2063      *
2064      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2065      *      boolean)
2066      *
2067      *      This method can be called from within the OSGi framework context,
2068      *      given the programming operation can take sometime, it not good
2069      *      pratice to have in it's context operations that can take time,
2070      *      hence moving off to a different thread for async processing.
2071      */
2072     private ExecutorService executor;
2073     @Override
2074     public void modeChangeNotify(final Node node, final boolean proactive) {
2075         Callable<Status> modeChangeCallable = new Callable<Status>() {
2076             @Override
2077             public Status call() throws Exception {
2078                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2079
2080                 List<String> puntAction = new ArrayList<String>();
2081                 puntAction.add(ActionType.CONTROLLER.toString());
2082
2083                 FlowConfig allowARP = new FlowConfig();
2084                 allowARP.setInstallInHw(true);
2085                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2086                 allowARP.setPriority("1");
2087                 allowARP.setNode(node);
2088                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2089                         .toUpperCase());
2090                 allowARP.setActions(puntAction);
2091                 defaultConfigs.add(allowARP);
2092
2093                 FlowConfig allowLLDP = new FlowConfig();
2094                 allowLLDP.setInstallInHw(true);
2095                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2096                 allowLLDP.setPriority("1");
2097                 allowLLDP.setNode(node);
2098                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2099                         .toUpperCase());
2100                 allowLLDP.setActions(puntAction);
2101                 defaultConfigs.add(allowLLDP);
2102
2103                 List<String> dropAction = new ArrayList<String>();
2104                 dropAction.add(ActionType.DROP.toString());
2105
2106                 FlowConfig dropAllConfig = new FlowConfig();
2107                 dropAllConfig.setInstallInHw(true);
2108                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2109                         + FlowConfig.INTERNALSTATICFLOWEND);
2110                 dropAllConfig.setPriority("0");
2111                 dropAllConfig.setNode(node);
2112                 dropAllConfig.setActions(dropAction);
2113                 defaultConfigs.add(dropAllConfig);
2114
2115                 log.trace("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2116                 for (FlowConfig fc : defaultConfigs) {
2117                     // check if the frm really needs to act on the notification.
2118                     // this is to check against duplicate notifications
2119                     if(programInternalFlow(proactive, fc)) {
2120                         Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2121                         if (status.isSuccess()) {
2122                             log.trace("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2123                         } else {
2124                             log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2125                                     fc.getName());
2126                         }
2127                     } else {
2128                         log.debug("Got redundant install request for internal flow: {} on node: {}. Request not sent to FRM.", fc.getName(), node);
2129                     }
2130                 }
2131                 return new Status(StatusCode.SUCCESS);
2132             }
2133         };
2134
2135         /*
2136          * Execute the work outside the caller context, this could be an
2137          * expensive operation and we don't want to block the caller for it.
2138          */
2139         this.executor.submit(modeChangeCallable);
2140     }
2141
2142     /**
2143      * Remove from the databases all the flows installed on the node
2144      *
2145      * @param node
2146      */
2147     private void cleanDatabaseForNode(Node node) {
2148         log.trace("Cleaning Flow database for Node {}", node);
2149         if (nodeFlows.containsKey(node)) {
2150             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2151
2152             for (FlowEntryInstall entry : toRemove) {
2153                 updateSwViews(entry, false);
2154             }
2155         }
2156     }
2157
2158     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2159         if (nc == null) {
2160             return false;
2161         }
2162
2163         Match match = flow.getMatch();
2164         if (match.isPresent(MatchType.IN_PORT)) {
2165             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2166             if (matchPort.equals(nc)) {
2167                 return true;
2168             }
2169         }
2170         List<Action> actionsList = flow.getActions();
2171         if (actionsList != null) {
2172             for (Action action : actionsList) {
2173                 if (action instanceof Output) {
2174                     NodeConnector actionPort = ((Output) action).getPort();
2175                     if (actionPort.equals(nc)) {
2176                         return true;
2177                     }
2178                 }
2179             }
2180         }
2181         return false;
2182     }
2183
2184     @Override
2185     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2186         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2187     }
2188
2189     @Override
2190     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2191         boolean updateStaticFlowCluster = false;
2192
2193         switch (type) {
2194         case ADDED:
2195             break;
2196         case CHANGED:
2197             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2198             if (config != null) {
2199                 switch (config.getValue()) {
2200                 case Config.ADMIN_DOWN:
2201                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2202                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2203                     break;
2204                 case Config.ADMIN_UP:
2205                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2206                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2207                     break;
2208                 case Config.ADMIN_UNDEF:
2209                     break;
2210                 default:
2211                 }
2212             }
2213             break;
2214         case REMOVED:
2215             // This is the case where a switch port is removed from the SDN agent space
2216             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2217             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2218             break;
2219         default:
2220
2221         }
2222
2223         if (updateStaticFlowCluster) {
2224             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2225         }
2226     }
2227
2228     /*
2229      * It goes through the static flows configuration, it identifies the ones
2230      * which have the specified node connector as input or output port and
2231      * install them on the network node if they are marked to be installed in
2232      * hardware and their status shows they were not installed yet
2233      */
2234     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2235         boolean updated = false;
2236         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2237         for (FlowConfig flowConfig : flowConfigForNode) {
2238             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2239                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(StatusCode.SUCCESS.toString())) {
2240                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2241                     if (!status.isSuccess()) {
2242                         flowConfig.setStatus(status.getDescription());
2243                     } else {
2244                         flowConfig.setStatus(StatusCode.SUCCESS.toString());
2245                     }
2246                     updated = true;
2247                 }
2248             }
2249         }
2250         return updated;
2251     }
2252
2253     /*
2254      * Remove from the network node all the flows which have the specified node
2255      * connector as input or output port. If any of the flow entry is a static
2256      * flow, it updates the correspondent configuration.
2257      */
2258     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2259         boolean updated = false;
2260         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2261         if (nodeFlowEntries == null) {
2262             return updated;
2263         }
2264         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2265             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2266                 Status status = this.removeEntryInternal(fei, true);
2267                 if (!status.isSuccess()) {
2268                     continue;
2269                 }
2270                 /*
2271                  * If the flow entry is a static flow, then update its
2272                  * configuration
2273                  */
2274                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2275                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2276                     if (flowConfig != null) {
2277                         flowConfig.setStatus(PORT_REMOVED);
2278                         updated = true;
2279                     }
2280                 }
2281             }
2282         }
2283         return updated;
2284     }
2285
2286     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2287         FlowConfig derivedFlow = new FlowConfig(original);
2288         derivedFlow.setDynamic(true);
2289         derivedFlow.setPortGroup(null);
2290         derivedFlow.setName(original.getName() + "_" + configName + "_" + port);
2291         derivedFlow.setIngressPort(port + "");
2292         return derivedFlow;
2293     }
2294
2295     private void addPortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2296         for (FlowConfig staticFlow : staticFlows.values()) {
2297             if (staticFlow.getPortGroup() == null) {
2298                 continue;
2299             }
2300             if ((staticFlow.getNode().equals(node)) && (staticFlow.getPortGroup().equals(config.getName()))) {
2301                 for (Short port : data.getPorts()) {
2302                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2303                     addStaticFlowInternal(derivedFlow, false);
2304                 }
2305             }
2306         }
2307     }
2308
2309     private void removePortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2310         for (FlowConfig staticFlow : staticFlows.values()) {
2311             if (staticFlow.getPortGroup() == null) {
2312                 continue;
2313             }
2314             if (staticFlow.getNode().equals(node) && staticFlow.getPortGroup().equals(config.getName())) {
2315                 for (Short port : data.getPorts()) {
2316                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2317                     removeStaticFlow(derivedFlow);
2318                 }
2319             }
2320         }
2321     }
2322
2323     @Override
2324     public void portGroupChanged(PortGroupConfig config, Map<Node, PortGroup> data, boolean add) {
2325         log.trace("PortGroup Changed for: {} Data: {}", config, portGroupData);
2326         Map<Node, PortGroup> existingData = portGroupData.get(config);
2327         if (existingData != null) {
2328             for (Map.Entry<Node, PortGroup> entry : data.entrySet()) {
2329                 PortGroup existingPortGroup = existingData.get(entry.getKey());
2330                 if (existingPortGroup == null) {
2331                     if (add) {
2332                         existingData.put(entry.getKey(), entry.getValue());
2333                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2334                     }
2335                 } else {
2336                     if (add) {
2337                         existingPortGroup.getPorts().addAll(entry.getValue().getPorts());
2338                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2339                     } else {
2340                         existingPortGroup.getPorts().removeAll(entry.getValue().getPorts());
2341                         removePortGroupFlows(config, entry.getKey(), entry.getValue());
2342                     }
2343                 }
2344             }
2345         } else {
2346             if (add) {
2347                 portGroupData.put(config, data);
2348                 for (Node swid : data.keySet()) {
2349                     addPortGroupFlows(config, swid, data.get(swid));
2350                 }
2351             }
2352         }
2353     }
2354
2355     @Override
2356     public boolean addPortGroupConfig(String name, String regex, boolean restore) {
2357         PortGroupConfig config = portGroupConfigs.get(name);
2358         if (config != null) {
2359             return false;
2360         }
2361
2362         if ((portGroupProvider == null) && !restore) {
2363             return false;
2364         }
2365         if ((portGroupProvider != null) && (!portGroupProvider.isMatchCriteriaSupported(regex))) {
2366             return false;
2367         }
2368
2369         config = new PortGroupConfig(name, regex);
2370         portGroupConfigs.put(name, config);
2371         if (portGroupProvider != null) {
2372             portGroupProvider.createPortGroupConfig(config);
2373         }
2374         return true;
2375     }
2376
2377     @Override
2378     public boolean delPortGroupConfig(String name) {
2379         PortGroupConfig config = portGroupConfigs.get(name);
2380         if (config == null) {
2381             return false;
2382         }
2383
2384         if (portGroupProvider != null) {
2385             portGroupProvider.deletePortGroupConfig(config);
2386         }
2387         portGroupConfigs.remove(name);
2388         return true;
2389     }
2390
2391     @Override
2392     public Map<String, PortGroupConfig> getPortGroupConfigs() {
2393         return portGroupConfigs;
2394     }
2395
2396     public boolean isPortGroupSupported() {
2397         if (portGroupProvider == null) {
2398             return false;
2399         }
2400         return true;
2401     }
2402
2403     public void setIContainer(IContainer s) {
2404         this.container = s;
2405     }
2406
2407     public void unsetIContainer(IContainer s) {
2408         if (this.container == s) {
2409             this.container = null;
2410         }
2411     }
2412
2413     public void setConfigurationContainerService(IConfigurationContainerService service) {
2414         log.trace("Got configuration service set request {}", service);
2415         this.configurationService = service;
2416     }
2417
2418     public void unsetConfigurationContainerService(IConfigurationContainerService service) {
2419         log.trace("Got configuration service UNset request");
2420         this.configurationService = null;
2421     }
2422
2423     @Override
2424     public PortGroupProvider getPortGroupProvider() {
2425         return portGroupProvider;
2426     }
2427
2428     public void unsetPortGroupProvider(PortGroupProvider portGroupProvider) {
2429         this.portGroupProvider = null;
2430     }
2431
2432     public void setPortGroupProvider(PortGroupProvider portGroupProvider) {
2433         this.portGroupProvider = portGroupProvider;
2434         portGroupProvider.registerPortGroupChange(this);
2435         for (PortGroupConfig config : portGroupConfigs.values()) {
2436             portGroupProvider.createPortGroupConfig(config);
2437         }
2438     }
2439
2440     public void setFrmAware(IForwardingRulesManagerAware obj) {
2441         this.frmAware.add(obj);
2442     }
2443
2444     public void unsetFrmAware(IForwardingRulesManagerAware obj) {
2445         this.frmAware.remove(obj);
2446     }
2447
2448     void setClusterContainerService(IClusterContainerServices s) {
2449         log.debug("Cluster Service set");
2450         this.clusterContainerService = s;
2451     }
2452
2453     void unsetClusterContainerService(IClusterContainerServices s) {
2454         if (this.clusterContainerService == s) {
2455             log.debug("Cluster Service removed!");
2456             this.clusterContainerService = null;
2457         }
2458     }
2459
2460     private String getContainerName() {
2461         if (container == null) {
2462             return GlobalConstants.DEFAULT.toString();
2463         }
2464         return container.getName();
2465     }
2466
2467     /**
2468      * Function called by the dependency manager when all the required
2469      * dependencies are satisfied
2470      *
2471      */
2472     void init() {
2473
2474         inContainerMode = false;
2475
2476         if (portGroupProvider != null) {
2477             portGroupProvider.registerPortGroupChange(this);
2478         }
2479
2480         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
2481         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
2482
2483         cacheStartup();
2484
2485         /*
2486          * If we are not the first cluster node to come up, do not initialize
2487          * the static flow entries ordinal
2488          */
2489         if (staticFlowsOrdinal.size() == 0) {
2490             staticFlowsOrdinal.put(0, Integer.valueOf(0));
2491         }
2492
2493         pendingEvents = new LinkedBlockingQueue<FRMEvent>();
2494
2495         // Initialize the event handler thread
2496         frmEventHandler = new Thread(new Runnable() {
2497             @Override
2498             public void run() {
2499                 while (!stopping) {
2500                     try {
2501                         final FRMEvent event = pendingEvents.take();
2502                         if (event == null) {
2503                             log.warn("Dequeued null event");
2504                             continue;
2505                         }
2506                         log.trace("Dequeued {} event", event.getClass().getSimpleName());
2507                         if (event instanceof NodeUpdateEvent) {
2508                             NodeUpdateEvent update = (NodeUpdateEvent) event;
2509                             Node node = update.getNode();
2510                             switch (update.getUpdateType()) {
2511                             case ADDED:
2512                                 addStaticFlowsToSwitch(node);
2513                                 break;
2514                             case REMOVED:
2515                                 cleanDatabaseForNode(node);
2516                                 updateStaticFlowConfigsOnNodeDown(node);
2517                                 break;
2518                             default:
2519                             }
2520                         } else if (event instanceof ErrorReportedEvent) {
2521                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
2522                             processErrorEvent(errEvent);
2523                         } else if (event instanceof WorkOrderEvent) {
2524                             /*
2525                              * Take care of handling the remote Work request
2526                              */
2527                             Runnable r = new Runnable() {
2528                                 @Override
2529                                 public void run() {
2530                                     WorkOrderEvent work = (WorkOrderEvent) event;
2531                                     FlowEntryDistributionOrder fe = work.getFe();
2532                                     if (fe != null) {
2533                                         logsync.trace("Executing the workOrder {}", fe);
2534                                         Status gotStatus = null;
2535                                         FlowEntryInstall feiCurrent = fe.getEntry();
2536                                         FlowEntryInstall feiNew = workOrder.get(fe);
2537                                         switch (fe.getUpType()) {
2538                                         case ADDED:
2539                                             gotStatus = addEntryInHw(feiCurrent, false);
2540                                             break;
2541                                         case CHANGED:
2542                                             gotStatus = modifyEntryInHw(feiCurrent, feiNew, false);
2543                                             break;
2544                                         case REMOVED:
2545                                             gotStatus = removeEntryInHw(feiCurrent, false);
2546                                             break;
2547                                         }
2548                                         // Remove the Order
2549                                         workOrder.remove(fe);
2550                                         logsync.trace(
2551                                                 "The workOrder has been executed and now the status is being returned {}", fe);
2552                                         // Place the status
2553                                         workStatus.put(fe, gotStatus);
2554                                     } else {
2555                                         log.warn("Not expected null WorkOrder", work);
2556                                     }
2557                                 }
2558                             };
2559                             if(executor != null) {
2560                                 executor.execute(r);
2561                             }
2562                         } else if (event instanceof WorkStatusCleanup) {
2563                             /*
2564                              * Take care of handling the remote Work request
2565                              */
2566                             WorkStatusCleanup work = (WorkStatusCleanup) event;
2567                             FlowEntryDistributionOrder fe = work.getFe();
2568                             if (fe != null) {
2569                                 logsync.trace("The workStatus {} is being removed", fe);
2570                                 workStatus.remove(fe);
2571                             } else {
2572                                 log.warn("Not expected null WorkStatus", work);
2573                             }
2574                         }  else if (event instanceof ContainerFlowChangeEvent) {
2575                             /*
2576                              * Whether it is an addition or removal, we have to
2577                              * recompute the merged flows entries taking into
2578                              * account all the current container flows because
2579                              * flow merging is not an injective function
2580                              */
2581                             updateFlowsContainerFlow();
2582                         } else if (event instanceof UpdateIndexDBs) {
2583                             UpdateIndexDBs update = (UpdateIndexDBs)event;
2584                             updateIndexDatabase(update.getFei(), update.isAddition());
2585                         } else {
2586                             log.warn("Dequeued unknown event {}", event.getClass().getSimpleName());
2587                         }
2588                     } catch (InterruptedException e) {
2589                         // clear pending events
2590                         pendingEvents.clear();
2591                     }
2592                 }
2593             }
2594         }, "FRM EventHandler Collector");
2595     }
2596
2597     /**
2598      * Function called by the dependency manager when at least one dependency
2599      * become unsatisfied or when the component is shutting down because for
2600      * example bundle is being stopped.
2601      *
2602      */
2603     void destroy() {
2604         // Interrupt the thread
2605         frmEventHandler.interrupt();
2606         // Clear the pendingEvents queue
2607         pendingEvents.clear();
2608         frmAware.clear();
2609         workMonitor.clear();
2610     }
2611
2612     /**
2613      * Function called by dependency manager after "init ()" is called and after
2614      * the services provided by the class are registered in the service registry
2615      *
2616      */
2617     void start() {
2618         /*
2619          * If running in default container, need to know if controller is in
2620          * container mode
2621          */
2622         if (GlobalConstants.DEFAULT.toString().equals(this.getContainerName())) {
2623             inContainerMode = containerManager.inContainerMode();
2624         }
2625
2626         // Initialize graceful stop flag
2627         stopping = false;
2628
2629         // Allocate the executor service
2630         this.executor = Executors.newFixedThreadPool(maxPoolSize);
2631
2632         // Start event handler thread
2633         frmEventHandler.start();
2634
2635         // replay the installedSwView data structure to populate
2636         // node flows and group flows
2637         for (FlowEntryInstall fei : installedSwView.values()) {
2638             pendingEvents.offer(new UpdateIndexDBs(fei, true));
2639         }
2640
2641         /*
2642          * Read startup and build database if we are the coordinator
2643          */
2644         loadFlowConfiguration();
2645     }
2646
2647     /**
2648      * Function called by the dependency manager before Container is Stopped and Destroyed.
2649      */
2650     public void containerStop() {
2651         uninstallAllFlowEntries(false);
2652     }
2653
2654     /**
2655      * Function called by the dependency manager before the services exported by
2656      * the component are unregistered, this will be followed by a "destroy ()"
2657      * calls
2658      */
2659     void stop() {
2660         stopping = true;
2661         // Shutdown executor
2662         this.executor.shutdownNow();
2663         // Now walk all the workMonitor and wake up the one sleeping because
2664         // destruction is happening
2665         for (FlowEntryDistributionOrder fe : workMonitor.keySet()) {
2666             FlowEntryDistributionOrderFutureTask task = workMonitor.get(fe);
2667             task.cancel(true);
2668         }
2669     }
2670
2671     public void setFlowProgrammerService(IFlowProgrammerService service) {
2672         this.programmer = service;
2673     }
2674
2675     public void unsetFlowProgrammerService(IFlowProgrammerService service) {
2676         if (this.programmer == service) {
2677             this.programmer = null;
2678         }
2679     }
2680
2681     public void setSwitchManager(ISwitchManager switchManager) {
2682         this.switchManager = switchManager;
2683     }
2684
2685     public void unsetSwitchManager(ISwitchManager switchManager) {
2686         if (this.switchManager == switchManager) {
2687             this.switchManager = null;
2688         }
2689     }
2690
2691     @Override
2692     public void tagUpdated(String containerName, Node n, short oldTag, short newTag, UpdateType t) {
2693         if (!container.getName().equals(containerName)) {
2694             return;
2695         }
2696     }
2697
2698     @Override
2699     public void containerFlowUpdated(String containerName, ContainerFlow previous, ContainerFlow current, UpdateType t) {
2700         if (!container.getName().equals(containerName)) {
2701             return;
2702         }
2703         log.trace("Container {}: Updating installed flows because of container flow change: {} {}",
2704                 container.getName(), t, current);
2705         ContainerFlowChangeEvent ev = new ContainerFlowChangeEvent(previous, current, t);
2706         pendingEvents.offer(ev);
2707     }
2708
2709     @Override
2710     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
2711         if (!container.getName().equals(containerName)) {
2712             return;
2713         }
2714
2715         boolean updateStaticFlowCluster = false;
2716
2717         switch (t) {
2718         case REMOVED:
2719             log.trace("Port {} was removed from container: uninstalling interested flows", nc);
2720             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nc);
2721             break;
2722         case ADDED:
2723             log.trace("Port {} was added to container: reinstall interested flows", nc);
2724             updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nc);
2725
2726             break;
2727         case CHANGED:
2728             break;
2729         default:
2730         }
2731
2732         if (updateStaticFlowCluster) {
2733             refreshClusterStaticFlowsStatus(nc.getNode());
2734         }
2735     }
2736
2737     @Override
2738     public void containerModeUpdated(UpdateType update) {
2739         // Only default container instance reacts on this event
2740         if (!container.getName().equals(GlobalConstants.DEFAULT.toString())) {
2741             return;
2742         }
2743         switch (update) {
2744         case ADDED:
2745             /*
2746              * Controller is moving to container mode. We are in the default
2747              * container context, we need to remove all our non-internal flows
2748              * to prevent any container isolation breakage. We also need to
2749              * preserve our flow so that they can be re-installed if we move
2750              * back to non container mode (no containers).
2751              */
2752             this.inContainerMode = true;
2753             this.uninstallAllFlowEntries(true);
2754             break;
2755         case REMOVED:
2756             this.inContainerMode = false;
2757             this.reinstallAllFlowEntries();
2758             break;
2759         default:
2760         }
2761
2762         // Update our configuration DB
2763         updateStaticFlowConfigsOnContainerModeChange(update);
2764     }
2765
2766     protected abstract class FRMEvent {
2767
2768     }
2769
2770     private class NodeUpdateEvent extends FRMEvent {
2771         private final Node node;
2772         private final UpdateType update;
2773
2774         public NodeUpdateEvent(UpdateType update, Node node) {
2775             this.update = update;
2776             this.node = node;
2777         }
2778
2779         public UpdateType getUpdateType() {
2780             return update;
2781         }
2782
2783         public Node getNode() {
2784             return node;
2785         }
2786     }
2787
2788     private class ErrorReportedEvent extends FRMEvent {
2789         private final long rid;
2790         private final Node node;
2791         private final Object error;
2792
2793         public ErrorReportedEvent(long rid, Node node, Object error) {
2794             this.rid = rid;
2795             this.node = node;
2796             this.error = error;
2797         }
2798
2799         public long getRequestId() {
2800             return rid;
2801         }
2802
2803         public Object getError() {
2804             return error;
2805         }
2806
2807         public Node getNode() {
2808             return node;
2809         }
2810     }
2811
2812     private class WorkOrderEvent extends FRMEvent {
2813         private FlowEntryDistributionOrder fe;
2814         private FlowEntryInstall newEntry;
2815
2816         /**
2817          * @param fe
2818          * @param newEntry
2819          */
2820         WorkOrderEvent(FlowEntryDistributionOrder fe, FlowEntryInstall newEntry) {
2821             this.fe = fe;
2822             this.newEntry = newEntry;
2823         }
2824
2825         /**
2826          * @return the fe
2827          */
2828         public FlowEntryDistributionOrder getFe() {
2829             return fe;
2830         }
2831
2832         /**
2833          * @return the newEntry
2834          */
2835         public FlowEntryInstall getNewEntry() {
2836             return newEntry;
2837         }
2838     }
2839     private class ContainerFlowChangeEvent extends FRMEvent {
2840         private final ContainerFlow previous;
2841         private final ContainerFlow current;
2842         private final UpdateType type;
2843
2844         public ContainerFlowChangeEvent(ContainerFlow previous, ContainerFlow current, UpdateType type) {
2845             this.previous = previous;
2846             this.current = current;
2847             this.type = type;
2848         }
2849
2850         public ContainerFlow getPrevious() {
2851             return this.previous;
2852         }
2853
2854         public ContainerFlow getCurrent() {
2855             return this.current;
2856         }
2857
2858         public UpdateType getType() {
2859             return this.type;
2860         }
2861     }
2862
2863
2864     private class WorkStatusCleanup extends FRMEvent {
2865         private FlowEntryDistributionOrder fe;
2866
2867         /**
2868          * @param fe
2869          */
2870         WorkStatusCleanup(FlowEntryDistributionOrder fe) {
2871             this.fe = fe;
2872         }
2873
2874         /**
2875          * @return the fe
2876          */
2877         public FlowEntryDistributionOrder getFe() {
2878             return fe;
2879         }
2880     }
2881
2882     private class UpdateIndexDBs extends FRMEvent {
2883         private FlowEntryInstall fei;
2884         private boolean add;
2885
2886         /**
2887          *
2888          * @param fei the flow entry which was installed/removed on the netwrok node
2889          * @param update
2890          */
2891         UpdateIndexDBs(FlowEntryInstall fei, boolean add) {
2892             this.fei = fei;
2893             this.add = add;
2894         }
2895
2896
2897         /**
2898          * @return the flowEntryInstall object which was added/removed
2899          * to/from the installed software view cache
2900          */
2901         public FlowEntryInstall getFei() {
2902             return fei;
2903         }
2904
2905         /**
2906          *
2907          * @return whether this was an flow addition or removal
2908          */
2909         public boolean isAddition() {
2910             return add;
2911         }
2912     }
2913
2914     @Override
2915     public Status saveConfiguration() {
2916         return saveConfig();
2917     }
2918
2919     @Override
2920     public void flowRemoved(Node node, Flow flow) {
2921         log.trace("Received flow removed notification on {} for {}", node, flow);
2922
2923         // For flow entry identification, only node, match and priority matter
2924         FlowEntryInstall test = new FlowEntryInstall(new FlowEntry("", "", flow, node), null);
2925         FlowEntryInstall installedEntry = this.installedSwView.get(test);
2926         if (installedEntry == null) {
2927             log.trace("Entry is not known to us");
2928             return;
2929         }
2930
2931         // Update Static flow status
2932         Integer key = 0;
2933         FlowConfig target = null;
2934         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2935             FlowConfig conf = entry.getValue();
2936             if (conf.isByNameAndNodeIdEqual(installedEntry.getFlowName(), node)) {
2937                 key = entry.getKey();
2938                 target = conf;
2939                 break;
2940             }
2941         }
2942         if (target != null) {
2943             // Update Configuration database
2944             if (target.getHardTimeout() != null || target.getIdleTimeout() != null) {
2945                 /*
2946                  * No need for checking if actual values: these strings were
2947                  * validated at configuration creation. Also, after a switch
2948                  * down scenario, no use to reinstall a timed flow. Mark it as
2949                  * "do not install". User can manually toggle it.
2950                  */
2951                 target.toggleInstallation();
2952             }
2953             target.setStatus(StatusCode.GONE.toString());
2954             staticFlows.put(key, target);
2955         }
2956
2957         // Update software views
2958         this.updateSwViews(installedEntry, false);
2959     }
2960
2961     @Override
2962     public void flowErrorReported(Node node, long rid, Object err) {
2963         log.trace("Got error {} for message rid {} from node {}", new Object[] { err, rid, node });
2964         pendingEvents.offer(new ErrorReportedEvent(rid, node, err));
2965     }
2966
2967     private void processErrorEvent(ErrorReportedEvent event) {
2968         Node node = event.getNode();
2969         long rid = event.getRequestId();
2970         Object error = event.getError();
2971         String errorString = (error == null) ? "Not provided" : error.toString();
2972         /*
2973          * If this was for a flow install, remove the corresponding entry from
2974          * the software view. If it was a Looking for the rid going through the
2975          * software database. TODO: A more efficient rid <-> FlowEntryInstall
2976          * mapping will have to be added in future
2977          */
2978         FlowEntryInstall target = null;
2979         List<FlowEntryInstall> flowEntryInstallList = nodeFlows.get(node);
2980         // flowEntryInstallList could be null.
2981         // so check for it.
2982         if(flowEntryInstallList != null) {
2983             for (FlowEntryInstall index : flowEntryInstallList) {
2984                 FlowEntryInstall entry = installedSwView.get(index);
2985                 if(entry != null) {
2986                     if (entry.getRequestId() == rid) {
2987                         target = entry;
2988                         break;
2989                     }
2990                 }
2991             }
2992         }
2993         if (target != null) {
2994             // This was a flow install, update database
2995             this.updateSwViews(target, false);
2996             // also update the config
2997             if(FlowConfig.STATICFLOWGROUP.equals(target.getGroupName())) {
2998                 ConcurrentMap.Entry<Integer, FlowConfig> staticFlowEntry = getStaticFlowEntry(target.getFlowName(),target.getNode());
2999                 // staticFlowEntry should never be null.
3000                 // the null check is just an extra defensive check.
3001                 if(staticFlowEntry != null) {
3002                     // Modify status and update cluster cache
3003                     log.debug("Updating static flow configuration on async error event");
3004                     String status = String.format("Cannot be installed on node. reason: %s", errorString);
3005                     staticFlowEntry.getValue().setStatus(status);
3006                     refreshClusterStaticFlowsStatus(node);
3007                 }
3008             }
3009         }
3010
3011         // Notify listeners
3012         if (frmAware != null) {
3013             synchronized (frmAware) {
3014                 for (IForwardingRulesManagerAware frma : frmAware) {
3015                     try {
3016                         frma.requestFailed(rid, errorString);
3017                     } catch (Exception e) {
3018                         log.warn("Failed to notify {}", frma);
3019                     }
3020                 }
3021             }
3022         }
3023     }
3024
3025     @Override
3026     public Status solicitStatusResponse(Node node, boolean blocking) {
3027         Status rv = new Status(StatusCode.INTERNALERROR);
3028
3029         if (this.programmer != null) {
3030             if (blocking) {
3031                 rv = programmer.syncSendBarrierMessage(node);
3032             } else {
3033                 rv = programmer.asyncSendBarrierMessage(node);
3034             }
3035         }
3036
3037         return rv;
3038     }
3039
3040     public void unsetIConnectionManager(IConnectionManager s) {
3041         if (s == this.connectionManager) {
3042             this.connectionManager = null;
3043         }
3044     }
3045
3046     public void setIConnectionManager(IConnectionManager s) {
3047         this.connectionManager = s;
3048     }
3049
3050     public void unsetIContainerManager(IContainerManager s) {
3051         if (s == this.containerManager) {
3052             this.containerManager = null;
3053         }
3054     }
3055
3056     public void setIContainerManager(IContainerManager s) {
3057         this.containerManager = s;
3058     }
3059
3060     @Override
3061     public void entryCreated(Object key, String cacheName, boolean originLocal) {
3062         /*
3063          * Do nothing
3064          */
3065     }
3066
3067     @Override
3068     public void entryUpdated(Object key, Object new_value, String cacheName, boolean originLocal) {
3069         /*
3070          * Streamline the updates for the per node and per group index databases
3071          */
3072         if (cacheName.equals(INSTALLED_SW_VIEW_CACHE)) {
3073             pendingEvents.offer(new UpdateIndexDBs((FlowEntryInstall)new_value, true));
3074         }
3075
3076         if (originLocal) {
3077             /*
3078              * Local updates are of no interest
3079              */
3080             return;
3081         }
3082         if (cacheName.equals(WORK_ORDER_CACHE)) {
3083             logsync.trace("Got a WorkOrderCacheUpdate for {}", key);
3084             /*
3085              * This is the case of one workOrder becoming available, so we need
3086              * to dispatch the work to the appropriate handler
3087              */
3088             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3089             FlowEntryInstall fei = fe.getEntry();
3090             if (fei == null) {
3091                 return;
3092             }
3093             Node n = fei.getNode();
3094             if (connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
3095                 logsync.trace("workOrder for fe {} processed locally", fe);
3096                 // I'm the controller in charge for the request, queue it for
3097                 // processing
3098                 pendingEvents.offer(new WorkOrderEvent(fe, (FlowEntryInstall) new_value));
3099             }
3100         } else if (cacheName.equals(WORK_STATUS_CACHE)) {
3101             logsync.trace("Got a WorkStatusCacheUpdate for {}", key);
3102             /*
3103              * This is the case of one workOrder being completed and a status
3104              * returned
3105              */
3106             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3107             /*
3108              * Check if the order was initiated by this controller in that case
3109              * we need to actually look at the status returned
3110              */
3111             if (fe.getRequestorController()
3112                     .equals(clusterContainerService.getMyAddress())) {
3113                 FlowEntryDistributionOrderFutureTask fet = workMonitor.remove(fe);
3114                 if (fet != null) {
3115                     logsync.trace("workStatus response is for us {}", fe);
3116                     // Signal we got the status
3117                     fet.gotStatus(fe, workStatus.get(fe));
3118                     pendingEvents.offer(new WorkStatusCleanup(fe));
3119                 }
3120             }
3121         }
3122     }
3123
3124     @Override
3125     public void entryDeleted(Object key, String cacheName, boolean originLocal) {
3126         /*
3127          * Streamline the updates for the per node and per group index databases
3128          */
3129         if (cacheName.equals(INSTALLED_SW_VIEW_CACHE)) {
3130             pendingEvents.offer(new UpdateIndexDBs((FlowEntryInstall)key, false));
3131         }
3132     }
3133
3134     /**
3135      * {@inheritDoc}
3136      */
3137     @Override
3138     public List<FlowEntry> getFlowEntriesForNode(Node node) {
3139         List<FlowEntry> list = new ArrayList<FlowEntry>();
3140         if (node != null) {
3141             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
3142                 if (node.equals(entry.getKey().getNode())) {
3143                     list.add(entry.getValue().clone());
3144                 }
3145             }
3146         }
3147         return list;
3148     }
3149
3150     /**
3151      * {@inheritDoc}
3152      */
3153     @Override
3154     public List<FlowEntry> getInstalledFlowEntriesForNode(Node node) {
3155         List<FlowEntry> list = new ArrayList<FlowEntry>();
3156         if (node != null) {
3157             List<FlowEntryInstall> flowEntryInstallList = this.nodeFlows.get(node);
3158             if(flowEntryInstallList != null) {
3159                 for(FlowEntryInstall fi: flowEntryInstallList) {
3160                     list.add(fi.getInstall().clone());
3161                 }
3162             }
3163         }
3164         return list;
3165     }
3166 }