opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManager.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.forwardingrulesmanager.internal;
  10
  11 import java.io.FileNotFoundException;
  12 import java.io.IOException;
  13 import java.io.ObjectInputStream;
  14 import java.util.ArrayList;
  15 import java.util.Collections;
  16 import java.util.EnumSet;
  17 import java.util.HashSet;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Map.Entry;
  21 import java.util.Set;
  22 import java.util.concurrent.BlockingQueue;
  23 import java.util.concurrent.Callable;
  24 import java.util.concurrent.ConcurrentHashMap;
  25 import java.util.concurrent.ConcurrentMap;
  26 import java.util.concurrent.ExecutionException;
  27 import java.util.concurrent.ExecutorService;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30
  31 import org.opendaylight.controller.clustering.services.CacheConfigException;
  32 import org.opendaylight.controller.clustering.services.CacheExistException;
  33 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
  34 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
  35 import org.opendaylight.controller.clustering.services.IClusterServices;
  36 import org.opendaylight.controller.configuration.ConfigurationObject;
  37 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
  38 import org.opendaylight.controller.configuration.IConfigurationContainerService;
  39 import org.opendaylight.controller.connectionmanager.IConnectionManager;
  40 import org.opendaylight.controller.containermanager.IContainerManager;
  41 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
  42 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
  43 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
  44 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  45 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
  46 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
  47 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
  48 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
  49 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
  50 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
  51 import org.opendaylight.controller.sal.action.Action;
  52 import org.opendaylight.controller.sal.action.ActionType;
  53 import org.opendaylight.controller.sal.action.Output;
  54 import org.opendaylight.controller.sal.connection.ConnectionLocality;
  55 import org.opendaylight.controller.sal.core.Config;
  56 import org.opendaylight.controller.sal.core.ContainerFlow;
  57 import org.opendaylight.controller.sal.core.IContainer;
  58 import org.opendaylight.controller.sal.core.IContainerLocalListener;
  59 import org.opendaylight.controller.sal.core.Node;
  60 import org.opendaylight.controller.sal.core.NodeConnector;
  61 import org.opendaylight.controller.sal.core.Property;
  62 import org.opendaylight.controller.sal.core.UpdateType;
  63 import org.opendaylight.controller.sal.flowprogrammer.Flow;
  64 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
  65 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
  66 import org.opendaylight.controller.sal.match.Match;
  67 import org.opendaylight.controller.sal.match.MatchType;
  68 import org.opendaylight.controller.sal.utils.EtherTypes;
  69 import org.opendaylight.controller.sal.utils.GlobalConstants;
  70 import org.opendaylight.controller.sal.utils.IObjectReader;
  71 import org.opendaylight.controller.sal.utils.Status;
  72 import org.opendaylight.controller.sal.utils.StatusCode;
  73 import org.opendaylight.controller.switchmanager.IInventoryListener;
  74 import org.opendaylight.controller.switchmanager.ISwitchManager;
  75 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
  76 import org.opendaylight.controller.switchmanager.Subnet;
  77 import org.slf4j.Logger;
  78 import org.slf4j.LoggerFactory;
  79
  80 /**
  81  * Class that manages forwarding rule installation and removal per container of
  82  * the network. It also maintains the central repository of all the forwarding
  83  * rules installed on the network nodes.
  84  */
  85 public class ForwardingRulesManager implements
  86         IForwardingRulesManager,
  87         PortGroupChangeListener,
  88         IContainerLocalListener,
  89         ISwitchManagerAware,
  90         IConfigurationContainerAware,
  91         IInventoryListener,
  92         IObjectReader,
  93         ICacheUpdateAware<Object,Object>,
  94         IFlowProgrammerListener {
  95
  96     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
  97     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
  98     private static final String PORT_REMOVED = "Port removed";
  99     private static final String NODE_DOWN = "Node is Down";
 100     private static final String INVALID_FLOW_ENTRY = "Invalid FlowEntry";
 101     private static final String STATIC_FLOWS_FILE_NAME = "frm_staticflows.conf";
 102     private static final String PORT_GROUP_FILE_NAME = "portgroup.conf";
 103     private ConcurrentMap<Integer, FlowConfig> staticFlows;
 104     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
 105     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
 106     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
 107     private ConcurrentMap<String, Object> TSPolicies;
 108     private IContainerManager containerManager;
 109     private IConfigurationContainerService configurationService;
 110     private boolean inContainerMode; // being used by global instance only
 111     protected boolean stopping;
 112
 113     /*
 114      * Flow database. It's the software view of what was requested to install
 115      * and what is installed on the switch. It is indexed by the entry itself.
 116      * The entry's hashcode resumes the network node index, the flow's priority
 117      * and the flow's match. The value element is a class which contains the
 118      * flow entry pushed by the applications modules and the respective
 119      * container flow merged version. In absence of container flows, the two
 120      * flow entries are the same.
 121      */
 122     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
 123     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
 124     /*
 125      * Per node and per group indexing
 126      */
 127     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
 128     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
 129
 130     /*
 131      * Inactive flow list. This is for the global instance of FRM It will
 132      * contain all the flow entries which were installed on the global container
 133      * when the first container is created.
 134      */
 135     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
 136
 137     private IContainer container;
 138     private Set<IForwardingRulesManagerAware> frmAware =
 139         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
 140     private PortGroupProvider portGroupProvider;
 141     private IFlowProgrammerService programmer;
 142     private IClusterContainerServices clusterContainerService = null;
 143     private ISwitchManager switchManager;
 144     private Thread frmEventHandler;
 145     protected BlockingQueue<FRMEvent> pendingEvents;
 146
 147     // Distributes FRM programming in the cluster
 148     private IConnectionManager connectionManager;
 149
 150     /*
 151      * Name clustered caches used to support FRM entry distribution these are by
 152      * necessity non-transactional as long as need to be able to synchronize
 153      * states also while a transaction is in progress
 154      */
 155     static final String WORK_ORDER_CACHE = "frm.workOrder";
 156     static final String WORK_STATUS_CACHE = "frm.workStatus";
 157     static final String ORIGINAL_SW_VIEW_CACHE = "frm.originalSwView";
 158     static final String INSTALLED_SW_VIEW_CACHE = "frm.installedSwView";
 159
 160     /*
 161      * Data structure responsible for distributing the FlowEntryInstall requests
 162      * in the cluster. The key value is entry that is being either Installed or
 163      * Updated or Delete. The value field is the same of the key value in case
 164      * of Installation or Deletion, it's the new entry in case of Modification,
 165      * this because the clustering caches don't allow null values.
 166      *
 167      * The logic behind this data structure is that the controller that initiate
 168      * the request will place the order here, someone will pick it and then will
 169      * remove from this data structure because is being served.
 170      *
 171      * TODO: We need to have a way to cleanup this data structure if entries are
 172      * not picked by anyone, which is always a case can happen especially on
 173      * Node disconnect cases.
 174      */
 175     protected ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
 176
 177     /*
 178      * Data structure responsible for retrieving the results of the workOrder
 179      * submitted to the cluster.
 180      *
 181      * The logic behind this data structure is that the controller that has
 182      * executed the order will then place the result in workStatus signaling
 183      * that there was a success or a failure.
 184      *
 185      * TODO: The workStatus entries need to have a lifetime associated in case
 186      * of requestor controller leaving the cluster.
 187      */
 188     protected ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
 189
 190     /*
 191      * Local Map used to hold the Future which a caller can use to monitor for
 192      * completion
 193      */
 194     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
 195             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
 196
 197     /*
 198      * Max pool size for the executor
 199      */
 200     private static final int maxPoolSize = 10;
 201
 202     /**
 203      * @param e
 204      *            Entry being installed/updated/removed
 205      * @param u
 206      *            New entry will be placed after the update operation. Valid
 207      *            only for UpdateType.CHANGED, null for all the other cases
 208      * @param t
 209      *            Type of update
 210      * @return a Future object for monitoring the progress of the result, or
 211      *         null in case the processing should take place locally
 212      */
 213     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
 214             UpdateType t) {
 215         // A null entry it's an unexpected condition, anyway it's safe to keep
 216         // the handling local
 217         if (e == null) {
 218             return null;
 219         }
 220
 221         Node n = e.getNode();
 222         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
 223             // Create the work order and distribute it
 224             FlowEntryDistributionOrder fe =
 225                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
 226             // First create the monitor job
 227             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
 228             logsync.trace("Node {} not local so sending fe {}", n, fe);
 229             workMonitor.put(fe, ret);
 230             if (t.equals(UpdateType.CHANGED)) {
 231                 // Then distribute the work
 232                 workOrder.put(fe, u);
 233             } else {
 234                 // Then distribute the work
 235                 workOrder.put(fe, e);
 236             }
 237             logsync.trace("WorkOrder requested");
 238             // Now create an Handle to monitor the execution of the operation
 239             return ret;
 240         }
 241
 242         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
 243         return null;
 244     }
 245
 246     /**
 247      * Adds a flow entry onto the network node It runs various validity checks
 248      * and derive the final container flows merged entries that will be
 249      * attempted to be installed
 250      *
 251      * @param flowEntry
 252      *            the original flow entry application requested to add
 253      * @param async
 254      *            the flag indicating if this is a asynchronous request
 255      * @return the status of this request. In case of asynchronous call, it will
 256      *         contain the unique id assigned to this request
 257      */
 258     private Status addEntry(FlowEntry flowEntry, boolean async) {
 259
 260         // Sanity Check
 261         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 262             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 263             log.warn(logMsg, flowEntry);
 264             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 265         }
 266
 267         /*
 268          * Redundant Check: Check if the request is a redundant one from the
 269          * same application the flowEntry is equal to an existing one. Given we
 270          * do not have an application signature in the requested FlowEntry yet,
 271          * we are here detecting the above condition by comparing the flow
 272          * names, if set. If they are equal to the installed flow, most likely
 273          * this is a redundant installation request from the same application
 274          * and we can silently return success
 275          *
 276          * TODO: in future a sort of application reference list mechanism will
 277          * be added to the FlowEntry so that exact flow can be used by different
 278          * applications.
 279          */
 280         FlowEntry present = this.originalSwView.get(flowEntry);
 281         if (present != null) {
 282             boolean sameFlow = present.getFlow().equals(flowEntry.getFlow());
 283             boolean sameApp = present.getFlowName() != null && present.getFlowName().equals(flowEntry.getFlowName());
 284             if (sameFlow && sameApp) {
 285                 log.trace("Skipping redundant request for flow {} on node {}", flowEntry.getFlowName(),
 286                         flowEntry.getNode());
 287                 return new Status(StatusCode.SUCCESS, "Entry is already present");
 288             }
 289         }
 290
 291         /*
 292          * Derive the container flow merged entries to install In presence of N
 293          * container flows, we may end up with N different entries to install...
 294          */
 295         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 296
 297         // Container Flow conflict Check
 298         if (toInstallList.isEmpty()) {
 299             String msg = "Flow Entry conflicts with all Container Flows";
 300             String logMsg = msg + ": {}";
 301             log.warn(logMsg, flowEntry);
 302             return new Status(StatusCode.CONFLICT, msg);
 303         }
 304
 305         // Derive the list of entries good to be installed
 306         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 307         for (FlowEntryInstall entry : toInstallList) {
 308             // Conflict Check: Verify new entry would not overwrite existing
 309             // ones
 310             if (this.installedSwView.containsKey(entry)) {
 311                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
 312                 log.trace("Aborting to install {}", entry);
 313                 continue;
 314             }
 315             toInstallSafe.add(entry);
 316         }
 317
 318         // Declare failure if all the container flow merged entries clash with
 319         // existing entries
 320         if (toInstallSafe.size() == 0) {
 321             String msg = "A flow with same match and priority exists on the target node";
 322             String logMsg = msg + ": {}";
 323             log.warn(logMsg, flowEntry);
 324             return new Status(StatusCode.CONFLICT, msg);
 325         }
 326
 327         // Try to install an entry at the time
 328         Status error = new Status(null, null);
 329         Status succeded = null;
 330         boolean oneSucceded = false;
 331         for (FlowEntryInstall installEntry : toInstallSafe) {
 332
 333             // Install and update database
 334             Status ret = addEntryInternal(installEntry, async);
 335
 336             if (ret.isSuccess()) {
 337                 oneSucceded = true;
 338                 /*
 339                  * The first successful status response will be returned For the
 340                  * asynchronous call, we can discard the container flow
 341                  * complication for now and assume we will always deal with one
 342                  * flow only per request
 343                  */
 344                 succeded = ret;
 345             } else {
 346                 error = ret;
 347                 log.trace("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
 348             }
 349         }
 350
 351         return (oneSucceded) ? succeded : error;
 352     }
 353
 354     /**
 355      * Given a flow entry and the list of container flows, it returns the list
 356      * of container flow merged flow entries good to be installed on this
 357      * container. If the list of container flows is null or empty, the install
 358      * entry list will contain only one entry, the original flow entry. If the
 359      * flow entry is congruent with all the N container flows, then the output
 360      * install entry list will contain N entries. If the output list is empty,
 361      * it means the passed flow entry conflicts with all the container flows.
 362      *
 363      * @param cFlowList
 364      *            The list of container flows
 365      * @return the list of container flow merged entries good to be installed on
 366      *         this container
 367      */
 368     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
 369         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
 370
 371         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
 372             // No container flows => entry good to be installed unchanged
 373             toInstallList.add(new FlowEntryInstall(request.clone(), null));
 374         } else {
 375             // Create the list of entries to be installed. If the flow entry is
 376             // not congruent with any container flow, no install entries will be
 377             // created
 378             for (ContainerFlow cFlow : container.getContainerFlows()) {
 379                 if (cFlow.allowsFlow(request.getFlow())) {
 380                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
 381                 }
 382             }
 383         }
 384         return toInstallList;
 385     }
 386
 387     /**
 388      * Modify a flow entry with a new one It runs various validity check and
 389      * derive the final container flows merged flow entries to work with
 390      *
 391      * @param currentFlowEntry
 392      * @param newFlowEntry
 393      * @param async
 394      *            the flag indicating if this is a asynchronous request
 395      * @return the status of this request. In case of asynchronous call, it will
 396      *         contain the unique id assigned to this request
 397      */
 398     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
 399         Status retExt;
 400
 401         // Sanity checks
 402         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
 403                 || newFlowEntry.getNode() == null || newFlowEntry.getFlow() == null) {
 404             String msg = "Modify: " + INVALID_FLOW_ENTRY;
 405             String logMsg = msg + ": {} or {}";
 406             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 407             return new Status(StatusCode.NOTACCEPTABLE, msg);
 408         }
 409         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
 410                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
 411             String msg = "Modify: Incompatible Flow Entries";
 412             String logMsg = msg + ": {} and {}";
 413             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 414             return new Status(StatusCode.NOTACCEPTABLE, msg);
 415         }
 416
 417         // Equality Check
 418         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
 419             String msg = "Modify skipped as flows are the same";
 420             String logMsg = msg + ": {} and {}";
 421             log.debug(logMsg, currentFlowEntry, newFlowEntry);
 422             return new Status(StatusCode.SUCCESS, msg);
 423         }
 424
 425         /*
 426          * Conflict Check: Verify the new entry would not conflict with an
 427          * existing one. This is a loose check on the previous original flow
 428          * entry requests. No check on the container flow merged flow entries
 429          * (if any) yet
 430          */
 431         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
 432         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
 433             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
 434             String logMsg = msg + ": {}";
 435             log.warn(logMsg, currentFlowEntry);
 436             return new Status(StatusCode.CONFLICT, msg);
 437         }
 438
 439         // Derive the installed and toInstall entries
 440         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
 441                 container.getContainerFlows());
 442         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
 443
 444         if (toInstallList.isEmpty()) {
 445             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
 446             String logMsg = msg + ": {}";
 447             log.warn(logMsg, newFlowEntry);
 448             log.warn(msg);
 449             return new Status(StatusCode.CONFLICT, msg);
 450         }
 451
 452         /*
 453          * If the two list sizes differ, it means the new flow entry does not
 454          * satisfy the same number of container flows the current entry does.
 455          * This is only possible when the new entry and current entry have
 456          * different match. In this scenario the modification would ultimately
 457          * be handled as a remove and add operations in the protocol plugin.
 458          *
 459          * Also, if any of the new flow entries would clash with an existing
 460          * one, we cannot proceed with the modify operation, because it would
 461          * fail for some entries and leave stale entries on the network node.
 462          * Modify path can be taken only if it can be performed completely, for
 463          * all entries.
 464          *
 465          * So, for the above two cases, to simplify, let's decouple the modify
 466          * in: 1) remove current entries 2) install new entries
 467          */
 468         Status succeeded = null;
 469         boolean decouple = false;
 470         if (installedList.size() != toInstallList.size()) {
 471             log.trace("Modify: New flow entry does not satisfy the same "
 472                     + "number of container flows as the original entry does");
 473             decouple = true;
 474         }
 475         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 476         for (FlowEntryInstall installEntry : toInstallList) {
 477             /*
 478              * Conflict Check: Verify the new entry would not overwrite another
 479              * existing one
 480              */
 481             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
 482             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
 483                 log.trace("Modify: new container flow merged flow entry clashes with existing flow");
 484                 decouple = true;
 485             } else {
 486                 toInstallSafe.add(installEntry);
 487             }
 488         }
 489
 490         if (decouple) {
 491             // Remove current entries
 492             for (FlowEntryInstall currEntry : installedList) {
 493                 this.removeEntryInternal(currEntry, async);
 494             }
 495             // Install new entries
 496             for (FlowEntryInstall newEntry : toInstallSafe) {
 497                 succeeded = this.addEntryInternal(newEntry, async);
 498             }
 499         } else {
 500             /*
 501              * The two list have the same size and the entries to install do not
 502              * clash with any existing flow on the network node. We assume here
 503              * (and might be wrong) that the same container flows that were
 504              * satisfied by the current entries are the same that are satisfied
 505              * by the new entries. Let's take the risk for now.
 506              *
 507              * Note: modification has to be complete. If any entry modification
 508              * fails, we need to stop, restore the already modified entries, and
 509              * declare failure.
 510              */
 511             Status retModify = null;
 512             int i = 0;
 513             int size = toInstallList.size();
 514             while (i < size) {
 515                 // Modify and update database
 516                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
 517                 if (retModify.isSuccess()) {
 518                     i++;
 519                 } else {
 520                     break;
 521                 }
 522             }
 523             // Check if uncompleted modify
 524             if (i < size) {
 525                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
 526                 // Restore original entries
 527                 int j = 0;
 528                 while (j < i) {
 529                     log.info("Attempting to restore initial entries");
 530                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
 531                     if (retExt.isSuccess()) {
 532                         j++;
 533                     } else {
 534                         break;
 535                     }
 536                 }
 537                 // Fatal error, recovery failed
 538                 if (j < i) {
 539                     String msg = "Flow recovery failed ! Unrecoverable Error";
 540                     log.error(msg);
 541                     return new Status(StatusCode.INTERNALERROR, msg);
 542                 }
 543             }
 544             succeeded = retModify;
 545         }
 546         /*
 547          * The first successful status response will be returned. For the
 548          * asynchronous call, we can discard the container flow complication for
 549          * now and assume we will always deal with one flow only per request
 550          */
 551         return succeeded;
 552     }
 553
 554     /**
 555      * This is the function that modifies the final container flows merged
 556      * entries on the network node and update the database. It expects that all
 557      * the validity checks are passed.
 558      * This function is supposed to be called only on the controller on which
 559      * the IFRM call is executed.
 560      *
 561      * @param currentEntries
 562      * @param newEntries
 563      * @param async
 564      *            the flag indicating if this is a asynchronous request
 565      * @return the status of this request. In case of asynchronous call, it will
 566      *         contain the unique id assigned to this request
 567      */
 568     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 569         Status status = new Status(StatusCode.UNDEFINED);
 570         FlowEntryDistributionOrderFutureTask futureStatus =
 571                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
 572         if (futureStatus != null) {
 573             try {
 574                 status = futureStatus.get();
 575                 if (status.getCode()
 576                         .equals(StatusCode.TIMEOUT)) {
 577                     // A timeout happened, lets cleanup the workMonitor
 578                     workMonitor.remove(futureStatus.getOrder());
 579                 }
 580             } catch (InterruptedException e) {
 581                 log.error("", e);
 582             } catch (ExecutionException e) {
 583                 log.error("", e);
 584             }
 585         } else {
 586             // Modify the flow on the network node
 587             status = modifyEntryInHw(currentEntries, newEntries, async);
 588         }
 589
 590         if (!status.isSuccess()) {
 591             log.trace("{} SDN Plugin failed to program the flow: {}. The failure is: {}",
 592                     (futureStatus != null) ? "Remote" : "Local", newEntries.getInstall(), status.getDescription());
 593             return status;
 594         }
 595
 596         log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
 597
 598         // Update DB
 599         newEntries.setRequestId(status.getRequestId());
 600         updateSwViews(currentEntries, false);
 601         updateSwViews(newEntries, true);
 602
 603         return status;
 604     }
 605
 606     private Status modifyEntryInHw(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 607         return async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall().getFlow(),
 608                 newEntries.getInstall().getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries
 609                 .getInstall().getFlow(), newEntries.getInstall().getFlow());
 610     }
 611
 612     /**
 613      * Remove a flow entry. If the entry is not present in the software view
 614      * (entry or node not present), it return successfully
 615      *
 616      * @param flowEntry
 617      *            the flow entry to remove
 618      * @param async
 619      *            the flag indicating if this is a asynchronous request
 620      * @return the status of this request. In case of asynchronous call, it will
 621      *         contain the unique id assigned to this request
 622      */
 623     private Status removeEntry(FlowEntry flowEntry, boolean async) {
 624         Status error = new Status(null, null);
 625
 626         // Sanity Check
 627         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 628             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 629             log.warn(logMsg, flowEntry);
 630             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 631         }
 632
 633         // Derive the container flows merged installed entries
 634         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 635
 636         Status succeeded = null;
 637         boolean atLeastOneRemoved = false;
 638         for (FlowEntryInstall entry : installedList) {
 639             if (!installedSwView.containsKey(entry)) {
 640                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
 641                 log.debug(logMsg, flowEntry);
 642                 if (installedList.size() == 1) {
 643                     // If we had only one entry to remove, we are done
 644                     return new Status(StatusCode.SUCCESS);
 645                 } else {
 646                     continue;
 647                 }
 648             }
 649
 650             // Remove and update DB
 651             Status ret = removeEntryInternal(entry, async);
 652
 653             if (!ret.isSuccess()) {
 654                 error = ret;
 655                 log.trace("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
 656                 if (installedList.size() == 1) {
 657                     // If we had only one entry to remove, this is fatal failure
 658                     return error;
 659                 }
 660             } else {
 661                 succeeded = ret;
 662                 atLeastOneRemoved = true;
 663             }
 664         }
 665
 666         /*
 667          * No worries if full removal failed. Consistency checker will take care
 668          * of removing the stale entries later, or adjusting the software
 669          * database if not in sync with hardware
 670          */
 671         return (atLeastOneRemoved) ? succeeded : error;
 672     }
 673
 674     /**
 675      * This is the function that removes the final container flows merged entry
 676      * from the network node and update the database. It expects that all the
 677      * validity checks are passed
 678      * This function is supposed to be called only on the controller on which
 679      * the IFRM call is executed.
 680      *
 681      * @param entry
 682      *            the flow entry to remove
 683      * @param async
 684      *            the flag indicating if this is a asynchronous request
 685      * @return the status of this request. In case of asynchronous call, it will
 686      *         contain the unique id assigned to this request
 687      */
 688     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
 689         Status status = new Status(StatusCode.UNDEFINED);
 690         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
 691         if (futureStatus != null) {
 692             try {
 693                 status = futureStatus.get();
 694                 if (status.getCode().equals(StatusCode.TIMEOUT)) {
 695                     // A timeout happened, lets cleanup the workMonitor
 696                     workMonitor.remove(futureStatus.getOrder());
 697                 }
 698             } catch (InterruptedException e) {
 699                 log.error("", e);
 700             } catch (ExecutionException e) {
 701                 log.error("", e);
 702             }
 703         } else {
 704             // Mark the entry to be deleted (for CC just in case we fail)
 705             entry.toBeDeleted();
 706
 707             // Remove from node
 708             status = removeEntryInHw(entry, async);
 709         }
 710
 711         if (!status.isSuccess()) {
 712             log.trace("{} SDN Plugin failed to remove the flow: {}. The failure is: {}",
 713                     (futureStatus != null) ? "Remote" : "Local", entry.getInstall(), status.getDescription());
 714             return status;
 715         }
 716
 717         log.trace("Removed  {}", entry.getInstall());
 718
 719         // Update DB
 720         updateSwViews(entry, false);
 721
 722         return status;
 723     }
 724
 725     private Status removeEntryInHw(FlowEntryInstall entry, boolean async) {
 726         return async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall().getFlow()) : programmer
 727                 .removeFlow(entry.getNode(), entry.getInstall().getFlow());
 728     }
 729
 730     /**
 731      * This is the function that installs the final container flow merged entry
 732      * on the network node and updates the database. It expects that all the
 733      * validity and conflict checks are passed. That means it does not check
 734      * whether this flow would conflict or overwrite an existing one.
 735      * This function is supposed to be called only on the controller on which
 736      * the IFRM call is executed.
 737      *
 738      * @param entry
 739      *            the flow entry to install
 740      * @param async
 741      *            the flag indicating if this is a asynchronous request
 742      * @return the status of this request. In case of asynchronous call, it will
 743      *         contain the unique id assigned to this request
 744      */
 745     private Status addEntryInternal(FlowEntryInstall entry, boolean async) {
 746         Status status = new Status(StatusCode.UNDEFINED);
 747         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
 748         if (futureStatus != null) {
 749             try {
 750                 status = futureStatus.get();
 751                 if (status.getCode().equals(StatusCode.TIMEOUT)) {
 752                     // A timeout happened, lets cleanup the workMonitor
 753                     workMonitor.remove(futureStatus.getOrder());
 754                 }
 755             } catch (InterruptedException e) {
 756                 log.error("", e);
 757             } catch (ExecutionException e) {
 758                 log.error("", e);
 759             }
 760         } else {
 761             status = addEntryInHw(entry, async);
 762         }
 763
 764         if (!status.isSuccess()) {
 765             log.trace("{} SDN Plugin failed to program the flow: {}. The failure is: {}",
 766                     (futureStatus != null) ? "Remote" : "Local", entry.getInstall(), status.getDescription());
 767             return status;
 768         }
 769
 770         log.trace("Added    {}", entry.getInstall());
 771
 772         // Update DB
 773         entry.setRequestId(status.getRequestId());
 774         updateSwViews(entry, true);
 775
 776         return status;
 777     }
 778
 779     private Status addEntryInHw(FlowEntryInstall entry, boolean async) {
 780         // Install the flow on the network node
 781         return async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall().getFlow()) : programmer.addFlow(
 782                 entry.getNode(), entry.getInstall().getFlow());
 783     }
 784
 785     /**
 786      * Returns true if the flow conflicts with all the container's flows. This
 787      * means that if the function returns true, the passed flow entry is
 788      * congruent with at least one container flow, hence it is good to be
 789      * installed on this container.
 790      *
 791      * @param flowEntry
 792      * @return true if flow conflicts with all the container flows, false
 793      *         otherwise
 794      */
 795     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
 796         List<ContainerFlow> cFlowList = container.getContainerFlows();
 797
 798         // Validity check and avoid unnecessary computation
 799         // Also takes care of default container where no container flows are
 800         // present
 801         if (cFlowList == null || cFlowList.isEmpty()) {
 802             return false;
 803         }
 804
 805         for (ContainerFlow cFlow : cFlowList) {
 806             if (cFlow.allowsFlow(flowEntry.getFlow())) {
 807                 // Entry is allowed by at least one container flow: good to go
 808                 return false;
 809             }
 810         }
 811         return true;
 812     }
 813
 814     private ConcurrentMap.Entry<Integer, FlowConfig> getStaticFlowEntry(String name, Node node) {
 815         for (ConcurrentMap.Entry<Integer, FlowConfig> flowEntry : staticFlows.entrySet()) {
 816             FlowConfig flowConfig = flowEntry.getValue();
 817             if (flowConfig.isByNameAndNodeIdEqual(name, node)) {
 818                 return flowEntry;
 819             }
 820         }
 821         return null;
 822     }
 823
 824     private void updateIndexDatabase(FlowEntryInstall entry, boolean add) {
 825         // Update node indexed flow database
 826         updateNodeFlowsDB(entry, add);
 827
 828         // Update group indexed flow database
 829         updateGroupFlowsDB(entry, add);
 830     }
 831
 832     /*
 833      * Update the node mapped flows database
 834      */
 835     private void updateSwViews(FlowEntryInstall flowEntries, boolean add) {
 836         if (add) {
 837             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
 838             installedSwView.put(flowEntries, flowEntries);
 839         } else {
 840             originalSwView.remove(flowEntries.getOriginal());
 841             installedSwView.remove(flowEntries);
 842         }
 843     }
 844
 845     /*
 846      * Update the node mapped flows database
 847      */
 848     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 849         Node node = flowEntries.getNode();
 850
 851         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
 852         if (nodeIndeces == null) {
 853             if (!add) {
 854                 return;
 855             } else {
 856                 nodeIndeces = new ArrayList<FlowEntryInstall>();
 857             }
 858         }
 859
 860         if (add) {
 861             // there may be an already existing entry.
 862             // remove it before adding the new one.
 863             // This is necessary since we have observed that in some cases
 864             // Infinispan does aggregation for operations (eg:- remove and then put a different value)
 865             // related to the same key within the same transaction.
 866             // Need this defensive code as the new FlowEntryInstall may be different
 867             // than the old one even though the equals method returns true. This is because
 868             // the equals method does not take into account the action list.
 869             if(nodeIndeces.contains(flowEntries)) {
 870                 nodeIndeces.remove(flowEntries);
 871             }
 872             nodeIndeces.add(flowEntries);
 873         } else {
 874             nodeIndeces.remove(flowEntries);
 875         }
 876
 877         // Update cache across cluster
 878         if (nodeIndeces.isEmpty()) {
 879             this.nodeFlows.remove(node);
 880         } else {
 881             this.nodeFlows.put(node, nodeIndeces);
 882         }
 883     }
 884
 885     /*
 886      * Update the group name mapped flows database
 887      */
 888     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 889         String groupName = flowEntries.getGroupName();
 890
 891         // Flow may not be part of a group
 892         if (groupName == null) {
 893             return;
 894         }
 895
 896         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
 897         if (indices == null) {
 898             if (!add) {
 899                 return;
 900             } else {
 901                 indices = new ArrayList<FlowEntryInstall>();
 902             }
 903         }
 904
 905         if (add) {
 906             // same comments in the similar code section in
 907             // updateNodeFlowsDB method apply here too
 908             if(indices.contains(flowEntries)) {
 909                 indices.remove(flowEntries);
 910             }
 911             indices.add(flowEntries);
 912         } else {
 913             indices.remove(flowEntries);
 914         }
 915
 916         // Update cache across cluster
 917         if (indices.isEmpty()) {
 918             this.groupFlows.remove(groupName);
 919         } else {
 920             this.groupFlows.put(groupName, indices);
 921         }
 922     }
 923
 924     /**
 925      * Remove a flow entry that has been added previously First checks if the
 926      * entry is effectively present in the local database
 927      */
 928     @SuppressWarnings("unused")
 929     private Status removeEntry(Node node, String flowName) {
 930         FlowEntryInstall target = null;
 931
 932         // Find in database
 933         for (FlowEntryInstall entry : installedSwView.values()) {
 934             if (entry.equalsByNodeAndName(node, flowName)) {
 935                 target = entry;
 936                 break;
 937             }
 938         }
 939
 940         // If it is not there, stop any further processing
 941         if (target == null) {
 942             return new Status(StatusCode.SUCCESS, "Entry is not present");
 943         }
 944
 945         // Remove from node
 946         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
 947
 948         // Update DB
 949         if (status.isSuccess()) {
 950             updateSwViews(target, false);
 951         } else {
 952             // log the error
 953             log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
 954                     status.getDescription());
 955         }
 956
 957         return status;
 958     }
 959
 960     @Override
 961     public Status installFlowEntry(FlowEntry flowEntry) {
 962         Status status;
 963         if (isContainerModeAllowed(flowEntry)) {
 964             status = addEntry(flowEntry, false);
 965         } else {
 966             String msg = "Controller in container mode: Install Refused";
 967             String logMsg = msg + ": {}";
 968             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 969             log.warn(logMsg, flowEntry);
 970         }
 971         return status;
 972     }
 973
 974     @Override
 975     public Status installFlowEntryAsync(FlowEntry flowEntry) {
 976         Status status;
 977         if (isContainerModeAllowed(flowEntry)) {
 978             status = addEntry(flowEntry, true);
 979         } else {
 980             String msg = "Controller in container mode: Install Refused";
 981             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 982             log.warn(msg);
 983         }
 984         return status;
 985     }
 986
 987     @Override
 988     public Status uninstallFlowEntry(FlowEntry flowEntry) {
 989         Status status;
 990         if (isContainerModeAllowed(flowEntry)) {
 991             status = removeEntry(flowEntry, false);
 992         } else {
 993             String msg = "Controller in container mode: Uninstall Refused";
 994             String logMsg = msg + ": {}";
 995             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 996             log.warn(logMsg, flowEntry);
 997         }
 998         return status;
 999     }
1000
1001     @Override
1002     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
1003         Status status;
1004         if (isContainerModeAllowed(flowEntry)) {
1005             status = removeEntry(flowEntry, true);
1006         } else {
1007             String msg = "Controller in container mode: Uninstall Refused";
1008             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1009             log.warn(msg);
1010         }
1011         return status;
1012     }
1013
1014     @Override
1015     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1016         Status status = null;
1017         if (isContainerModeAllowed(currentFlowEntry)) {
1018             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1019         } else {
1020             String msg = "Controller in container mode: Modify Refused";
1021             String logMsg = msg + ": {}";
1022             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1023             log.warn(logMsg, newFlowEntry);
1024         }
1025         return status;
1026     }
1027
1028     @Override
1029     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1030         Status status = null;
1031         if (isContainerModeAllowed(currentFlowEntry)) {
1032             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
1033         } else {
1034             String msg = "Controller in container mode: Modify Refused";
1035             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1036             log.warn(msg);
1037         }
1038         return status;
1039     }
1040
1041     /**
1042      * Returns whether the specified flow entry is allowed to be
1043      * installed/removed/modified based on the current container mode status.
1044      * This call always returns true in the container instance of forwarding
1045      * rules manager. It is meant for the global instance only (default
1046      * container) of forwarding rules manager. Idea is that for assuring
1047      * container isolation of traffic, flow installation in default container is
1048      * blocked when in container mode (containers are present). The only flows
1049      * that are allowed in container mode in the default container are the
1050      * proactive flows, the ones automatically installed on the network node
1051      * which forwarding mode has been configured to "proactive". These flows are
1052      * needed by controller to discover the nodes topology and to discover the
1053      * attached hosts for some SDN switches.
1054      *
1055      * @param flowEntry
1056      *            The flow entry to be installed/removed/modified
1057      * @return true if not in container mode or if flowEntry is internally
1058      *         generated
1059      */
1060     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1061         return (!inContainerMode) ? true : flowEntry.isInternal();
1062     }
1063
1064     @Override
1065     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1066         /*
1067          * Run a check on the original entries to decide whether to go with a
1068          * add or modify method. A loose check means only check against the
1069          * original flow entry requests and not against the installed flow
1070          * entries which are the result of the original entry merged with the
1071          * container flow(s) (if any). The modifyFlowEntry method in presence of
1072          * conflicts with the Container flows (if any) would revert back to a
1073          * delete + add pattern
1074          */
1075         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1076
1077         if (currentFlowEntry != null) {
1078             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1079         } else {
1080             return installFlowEntry(newFlowEntry);
1081         }
1082     }
1083
1084     @Override
1085     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1086         /*
1087          * Run a check on the original entries to decide whether to go with a
1088          * add or modify method. A loose check means only check against the
1089          * original flow entry requests and not against the installed flow
1090          * entries which are the result of the original entry merged with the
1091          * container flow(s) (if any). The modifyFlowEntry method in presence of
1092          * conflicts with the Container flows (if any) would revert back to a
1093          * delete + add pattern
1094          */
1095         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1096
1097         if (currentFlowEntry != null) {
1098             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1099         } else {
1100             return installFlowEntryAsync(newFlowEntry);
1101         }
1102     }
1103
1104     @Override
1105     public Status uninstallFlowEntryGroup(String groupName) {
1106         if (groupName == null || groupName.isEmpty()) {
1107             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1108         }
1109         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1110             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1111         }
1112         if (inContainerMode) {
1113             String msg = "Controller in container mode: Group Uninstall Refused";
1114             String logMsg = msg + ": {}";
1115             log.warn(logMsg, groupName);
1116             return new Status(StatusCode.NOTACCEPTABLE, msg);
1117         }
1118         int toBeRemoved = 0;
1119         String error = "";
1120         if (groupFlows.containsKey(groupName)) {
1121             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1122             toBeRemoved = list.size();
1123             for (FlowEntryInstall entry : list) {
1124                 // since this is the entry that was stored in groupFlows
1125                 // it is already validated and merged
1126                 // so can call removeEntryInternal directly
1127                 Status status = this.removeEntryInternal(entry, false);
1128                 if (status.isSuccess()) {
1129                     toBeRemoved -= 1;
1130                 } else {
1131                     error = status.getDescription();
1132                 }
1133             }
1134         }
1135         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1136                 "Not all the flows were removed: " + error);
1137     }
1138
1139     @Override
1140     public Status uninstallFlowEntryGroupAsync(String groupName) {
1141         if (groupName == null || groupName.isEmpty()) {
1142             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1143         }
1144         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1145             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1146         }
1147         if (inContainerMode) {
1148             String msg = "Controller in container mode: Group Uninstall Refused";
1149             String logMsg = msg + ": {}";
1150             log.warn(logMsg, groupName);
1151             return new Status(StatusCode.NOTACCEPTABLE, msg);
1152         }
1153         if (groupFlows.containsKey(groupName)) {
1154             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1155             for (FlowEntryInstall entry : list) {
1156                 this.removeEntry(entry.getOriginal(), true);
1157             }
1158         }
1159         return new Status(StatusCode.SUCCESS);
1160     }
1161
1162     @Override
1163     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1164         return entryConflictsWithContainerFlows(flowEntry);
1165     }
1166
1167     /**
1168      * Updates all installed flows because the container flow got updated This
1169      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1170      * Reinstall of all flows This is needed because a new container flows
1171      * merged flow may conflict with an existing old container flows merged flow
1172      * on the network node
1173      */
1174     protected void updateFlowsContainerFlow() {
1175         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1176         // First remove all installed entries
1177         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1178             FlowEntryInstall current = entry.getValue();
1179             // Store the original entry
1180             toReInstall.add(current.getOriginal());
1181             // Remove the old couples. No validity checks to be run, use the
1182             // internal remove
1183             this.removeEntryInternal(current, false);
1184         }
1185         // Then reinstall the original entries
1186         for (FlowEntry entry : toReInstall) {
1187             // Reinstall the original flow entries, via the regular path: new
1188             // cFlow merge + validations
1189             this.installFlowEntry(entry);
1190         }
1191     }
1192
1193     private void nonClusterObjectCreate() {
1194         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1195         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1196         TSPolicies = new ConcurrentHashMap<String, Object>();
1197         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1198         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1199         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1200         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1201         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1202     }
1203
1204     @Override
1205     public void setTSPolicyData(String policyname, Object o, boolean add) {
1206
1207         if (add) {
1208             /* Check if this policy already exists */
1209             if (!(TSPolicies.containsKey(policyname))) {
1210                 TSPolicies.put(policyname, o);
1211             }
1212         } else {
1213             TSPolicies.remove(policyname);
1214         }
1215         if (frmAware != null) {
1216             synchronized (frmAware) {
1217                 for (IForwardingRulesManagerAware frma : frmAware) {
1218                     try {
1219                         frma.policyUpdate(policyname, add);
1220                     } catch (Exception e) {
1221                         log.warn("Exception on callback", e);
1222                     }
1223                 }
1224             }
1225         }
1226     }
1227
1228     @Override
1229     public Map<String, Object> getTSPolicyData() {
1230         return TSPolicies;
1231     }
1232
1233     @Override
1234     public Object getTSPolicyData(String policyName) {
1235         if (TSPolicies.containsKey(policyName)) {
1236             return TSPolicies.get(policyName);
1237         } else {
1238             return null;
1239         }
1240     }
1241
1242     @Override
1243     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1244         List<FlowEntry> list = new ArrayList<FlowEntry>();
1245         if (policyName != null && !policyName.trim().isEmpty()) {
1246             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1247                 if (policyName.equals(entry.getKey().getGroupName())) {
1248                     list.add(entry.getValue().clone());
1249                 }
1250             }
1251         }
1252         return list;
1253     }
1254
1255     @Override
1256     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1257         List<FlowEntry> list = new ArrayList<FlowEntry>();
1258         if (policyName != null && !policyName.trim().isEmpty()) {
1259             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1260                 if (policyName.equals(entry.getKey().getGroupName())) {
1261                     list.add(entry.getValue().getInstall().clone());
1262                 }
1263             }
1264         }
1265         return list;
1266     }
1267
1268     @Override
1269     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1270
1271         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1272             if (flow.getFlowName().equals(flowName)) {
1273                 FlowEntry currentFlowEntry = flow.getOriginal();
1274                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1275                 for (NodeConnector dstPort : portList) {
1276                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1277                 }
1278                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1279                 if (error.isSuccess()) {
1280                     log.trace("Ports {} added to FlowEntry {}", portList, flowName);
1281                 } else {
1282                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1283                             currentFlowEntry.toString(), error.getDescription());
1284                 }
1285                 return;
1286             }
1287         }
1288         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1289     }
1290
1291     @Override
1292     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1293         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1294             FlowEntryInstall flow = this.installedSwView.get(index);
1295             if (flow.getFlowName().equals(flowName)) {
1296                 FlowEntry currentFlowEntry = flow.getOriginal();
1297                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1298                 for (NodeConnector dstPort : portList) {
1299                     Action action = new Output(dstPort);
1300                     newFlowEntry.getFlow().removeAction(action);
1301                 }
1302                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1303                 if (status.isSuccess()) {
1304                     log.trace("Ports {} removed from FlowEntry {}", portList, flowName);
1305                 } else {
1306                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1307                             currentFlowEntry.toString(), status.getDescription());
1308                 }
1309                 return;
1310             }
1311         }
1312         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1313     }
1314
1315     /*
1316      * This function assumes the target flow has only one output port
1317      */
1318     @Override
1319     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1320         FlowEntry currentFlowEntry = null;
1321         FlowEntry newFlowEntry = null;
1322
1323         // Find the flow
1324         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1325             FlowEntryInstall flow = this.installedSwView.get(index);
1326             if (flow.getFlowName().equals(flowName)) {
1327                 currentFlowEntry = flow.getOriginal();
1328                 break;
1329             }
1330         }
1331         if (currentFlowEntry == null) {
1332             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1333             return;
1334         }
1335
1336         // Create a flow copy with the new output port
1337         newFlowEntry = currentFlowEntry.clone();
1338         Action target = null;
1339         for (Action action : newFlowEntry.getFlow().getActions()) {
1340             if (action.getType() == ActionType.OUTPUT) {
1341                 target = action;
1342                 break;
1343             }
1344         }
1345         newFlowEntry.getFlow().removeAction(target);
1346         newFlowEntry.getFlow().addAction(new Output(outPort));
1347
1348         // Modify on network node
1349         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1350
1351         if (status.isSuccess()) {
1352             log.trace("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1353         } else {
1354             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1355                     status.getDescription());
1356         }
1357         return;
1358     }
1359
1360     @Override
1361     public NodeConnector getOutputPort(Node node, String flowName) {
1362         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1363             FlowEntryInstall flow = this.installedSwView.get(index);
1364             if (flow.getFlowName().equals(flowName)) {
1365                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1366                     if (action.getType() == ActionType.OUTPUT) {
1367                         return ((Output) action).getPort();
1368                     }
1369                 }
1370             }
1371         }
1372         return null;
1373     }
1374
1375     private void cacheStartup() {
1376         allocateCaches();
1377         retrieveCaches();
1378     }
1379
1380     private void allocateCaches() {
1381         if (this.clusterContainerService == null) {
1382             log.warn("Un-initialized clusterContainerService, can't create cache");
1383             return;
1384         }
1385
1386         log.debug("Allocating caches for Container {}", container.getName());
1387
1388         try {
1389             clusterContainerService.createCache(ORIGINAL_SW_VIEW_CACHE,
1390                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1391
1392             clusterContainerService.createCache(INSTALLED_SW_VIEW_CACHE,
1393                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1394
1395             clusterContainerService.createCache("frm.inactiveFlows",
1396                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1397
1398             clusterContainerService.createCache("frm.staticFlows",
1399                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1400
1401             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1402                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1403
1404             clusterContainerService.createCache("frm.portGroupConfigs",
1405                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1406
1407             clusterContainerService.createCache("frm.portGroupData",
1408                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1409
1410             clusterContainerService.createCache("frm.TSPolicies",
1411                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1412
1413             clusterContainerService.createCache(WORK_STATUS_CACHE,
1414                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1415
1416             clusterContainerService.createCache(WORK_ORDER_CACHE,
1417                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1418
1419         } catch (CacheConfigException cce) {
1420             log.error("CacheConfigException");
1421         } catch (CacheExistException cce) {
1422             log.error("CacheExistException");
1423         }
1424     }
1425
1426     @SuppressWarnings({ "unchecked" })
1427     private void retrieveCaches() {
1428         ConcurrentMap<?, ?> map;
1429
1430         if (this.clusterContainerService == null) {
1431             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1432             nonClusterObjectCreate();
1433             return;
1434         }
1435
1436         log.debug("Retrieving Caches for Container {}", container.getName());
1437
1438         map = clusterContainerService.getCache(ORIGINAL_SW_VIEW_CACHE);
1439         if (map != null) {
1440             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1441         } else {
1442             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1443         }
1444
1445         map = clusterContainerService.getCache(INSTALLED_SW_VIEW_CACHE);
1446         if (map != null) {
1447             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1448         } else {
1449             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1450         }
1451
1452         map = clusterContainerService.getCache("frm.inactiveFlows");
1453         if (map != null) {
1454             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1455         } else {
1456             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1457         }
1458
1459         map = clusterContainerService.getCache("frm.staticFlows");
1460         if (map != null) {
1461             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1462         } else {
1463             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1464         }
1465
1466         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1467         if (map != null) {
1468             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1469         } else {
1470             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1471         }
1472
1473         map = clusterContainerService.getCache("frm.portGroupConfigs");
1474         if (map != null) {
1475             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1476         } else {
1477             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1478         }
1479
1480         map = clusterContainerService.getCache("frm.portGroupData");
1481         if (map != null) {
1482             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1483         } else {
1484             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1485         }
1486
1487         map = clusterContainerService.getCache("frm.TSPolicies");
1488         if (map != null) {
1489             TSPolicies = (ConcurrentMap<String, Object>) map;
1490         } else {
1491             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1492         }
1493
1494         map = clusterContainerService.getCache(WORK_ORDER_CACHE);
1495         if (map != null) {
1496             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1497         } else {
1498             log.error("Retrieval of " + WORK_ORDER_CACHE + " cache failed for Container {}", container.getName());
1499         }
1500
1501         map = clusterContainerService.getCache(WORK_STATUS_CACHE);
1502         if (map != null) {
1503             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1504         } else {
1505             log.error("Retrieval of " + WORK_STATUS_CACHE + " cache failed for Container {}", container.getName());
1506         }
1507     }
1508
1509     private boolean flowConfigExists(FlowConfig config) {
1510         // Flow name has to be unique on per node id basis
1511         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1512             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1513                 return true;
1514             }
1515         }
1516         return false;
1517     }
1518
1519     @Override
1520     public Status addStaticFlow(FlowConfig config) {
1521         // Configuration object validation
1522         Status status = config.validate(container);
1523         if (!status.isSuccess()) {
1524             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1525             String error = "Invalid Configuration (" + status.getDescription() + ")";
1526             config.setStatus(error);
1527             return new Status(StatusCode.BADREQUEST, error);
1528         }
1529         return addStaticFlowInternal(config, false);
1530     }
1531
1532     /**
1533      * Private method to add a static flow configuration which does not run any
1534      * validation on the passed FlowConfig object. If restore is set to true,
1535      * configuration is stored in configuration database regardless the
1536      * installation on the network node was successful. This is useful at boot
1537      * when static flows are present in startup configuration and are read
1538      * before the switches connects.
1539      *
1540      * @param config
1541      *            The static flow configuration
1542      * @param restore
1543      *            if true, the configuration is stored regardless the
1544      *            installation on the network node was successful
1545      * @return The status of this request
1546      */
1547     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1548         boolean multipleFlowPush = false;
1549         String error;
1550         Status status;
1551         config.setStatus(StatusCode.SUCCESS.toString());
1552
1553         // Presence check
1554         if (flowConfigExists(config)) {
1555             error = "Entry with this name on specified switch already exists";
1556             log.warn("Entry with this name on specified switch already exists: {}", config);
1557             config.setStatus(error);
1558             return new Status(StatusCode.CONFLICT, error);
1559         }
1560
1561         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1562             for (String portGroupName : portGroupConfigs.keySet()) {
1563                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1564                     multipleFlowPush = true;
1565                     break;
1566                 }
1567             }
1568             if (!multipleFlowPush) {
1569                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1570                 error = "Invalid Configuration (Invalid PortGroup Name)";
1571                 config.setStatus(error);
1572                 return new Status(StatusCode.BADREQUEST, error);
1573             }
1574         }
1575
1576         /*
1577          * If requested program the entry in hardware first before updating the
1578          * StaticFlow DB
1579          */
1580         if (!multipleFlowPush) {
1581             // Program hw
1582             if (config.installInHw()) {
1583                 FlowEntry entry = config.getFlowEntry();
1584                 status = this.installFlowEntry(entry);
1585                 if (!status.isSuccess()) {
1586                     config.setStatus(status.getDescription());
1587                     if (!restore) {
1588                         return status;
1589                     }
1590                 }
1591             }
1592         }
1593
1594         /*
1595          * When the control reaches this point, either of the following
1596          * conditions is true 1. This is a single entry configuration (non
1597          * PortGroup) and the hardware installation is successful 2. This is a
1598          * multiple entry configuration (PortGroup) and hardware installation is
1599          * NOT done directly on this event. 3. The User prefers to retain the
1600          * configuration in Controller and skip hardware installation.
1601          *
1602          * Hence it is safe to update the StaticFlow DB at this point.
1603          *
1604          * Note : For the case of PortGrouping, it is essential to have this DB
1605          * populated before the PortGroupListeners can query for the DB
1606          * triggered using portGroupChanged event...
1607          */
1608         Integer ordinal = staticFlowsOrdinal.get(0);
1609         staticFlowsOrdinal.put(0, ++ordinal);
1610         staticFlows.put(ordinal, config);
1611
1612         if (multipleFlowPush) {
1613             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1614             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1615             if (existingData != null) {
1616                 portGroupChanged(pgconfig, existingData, true);
1617             }
1618         }
1619         return new Status(StatusCode.SUCCESS);
1620     }
1621
1622     private void addStaticFlowsToSwitch(Node node) {
1623         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1624             FlowConfig config = entry.getValue();
1625             if (config.isPortGroupEnabled()) {
1626                 continue;
1627             }
1628             if (config.getNode().equals(node)) {
1629                 if (config.installInHw() && !config.getStatus().equals(StatusCode.SUCCESS.toString())) {
1630                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1631                     config.setStatus(status.getDescription());
1632                 }
1633             }
1634         }
1635         // Update cluster cache
1636         refreshClusterStaticFlowsStatus(node);
1637     }
1638
1639     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1640         log.trace("Updating Static Flow configs on node down: {}", node);
1641
1642         List<Integer> toRemove = new ArrayList<Integer>();
1643         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1644
1645             FlowConfig config = entry.getValue();
1646
1647             if (config.isPortGroupEnabled()) {
1648                 continue;
1649             }
1650
1651             if (config.installInHw() && config.getNode().equals(node)) {
1652                 if (config.isInternalFlow()) {
1653                     // Take note of this controller generated static flow
1654                     toRemove.add(entry.getKey());
1655                 } else {
1656                     config.setStatus(NODE_DOWN);
1657                 }
1658             }
1659         }
1660         // Remove controller generated static flows for this node
1661         for (Integer index : toRemove) {
1662             staticFlows.remove(index);
1663         }
1664         // Update cluster cache
1665         refreshClusterStaticFlowsStatus(node);
1666
1667     }
1668
1669     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1670         log.trace("Updating Static Flow configs on container mode change: {}", update);
1671
1672         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1673             FlowConfig config = entry.getValue();
1674             if (config.isPortGroupEnabled()) {
1675                 continue;
1676             }
1677             if (config.installInHw() && !config.isInternalFlow()) {
1678                 switch (update) {
1679                 case ADDED:
1680                     config.setStatus("Removed from node because in container mode");
1681                     break;
1682                 case REMOVED:
1683                     config.setStatus(StatusCode.SUCCESS.toString());
1684                     break;
1685                 default:
1686                 }
1687             }
1688         }
1689         // Update cluster cache
1690         refreshClusterStaticFlowsStatus(null);
1691     }
1692
1693     @Override
1694     public Status removeStaticFlow(FlowConfig config) {
1695         /*
1696          * No config.isInternal() check as NB does not take this path and GUI
1697          * cannot issue a delete on an internal generated flow. We need this
1698          * path to be accessible when switch mode is changed from proactive to
1699          * reactive, so that we can remove the internal generated LLDP and ARP
1700          * punt flows
1701          */
1702
1703         // Look for the target configuration entry
1704         Integer key = 0;
1705         FlowConfig target = null;
1706         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1707             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1708                 key = entry.getKey();
1709                 target = entry.getValue();
1710                 break;
1711             }
1712         }
1713         if (target == null) {
1714             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1715         }
1716
1717         // Program the network node
1718         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1719
1720         // Update configuration database if programming was successful
1721         if (status.isSuccess()) {
1722             staticFlows.remove(key);
1723         }
1724
1725         return status;
1726     }
1727
1728     @Override
1729     public Status removeStaticFlow(String name, Node node) {
1730         // Look for the target configuration entry
1731         Integer key = 0;
1732         FlowConfig target = null;
1733         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1734             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1735                 key = mapEntry.getKey();
1736                 target = mapEntry.getValue();
1737                 break;
1738             }
1739         }
1740         if (target == null) {
1741             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1742         }
1743
1744         // Validity check for api3 entry point
1745         if (target.isInternalFlow()) {
1746             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1747             String logMsg = msg + ": {}";
1748             log.warn(logMsg, name);
1749             return new Status(StatusCode.NOTACCEPTABLE, msg);
1750         }
1751
1752         if (target.isPortGroupEnabled()) {
1753             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1754             String logMsg = msg + ": {}";
1755             log.warn(logMsg, name);
1756             return new Status(StatusCode.NOTACCEPTABLE, msg);
1757         }
1758
1759         // Program the network node
1760         Status status = this.removeEntry(target.getFlowEntry(), false);
1761
1762         // Update configuration database if programming was successful
1763         if (status.isSuccess()) {
1764             staticFlows.remove(key);
1765         }
1766
1767         return status;
1768     }
1769
1770     @Override
1771     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1772         // Validity check for api3 entry point
1773         if (newFlowConfig.isInternalFlow()) {
1774             String msg = "Invalid operation: Controller generated flow cannot be modified";
1775             String logMsg = msg + ": {}";
1776             log.warn(logMsg, newFlowConfig);
1777             return new Status(StatusCode.NOTACCEPTABLE, msg);
1778         }
1779
1780         // Validity Check
1781         Status status = newFlowConfig.validate(container);
1782         if (!status.isSuccess()) {
1783             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1784             newFlowConfig.setStatus(msg);
1785             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1786             return new Status(StatusCode.BADREQUEST, msg);
1787         }
1788
1789         FlowConfig oldFlowConfig = null;
1790         Integer index = null;
1791         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1792             FlowConfig entry = mapEntry.getValue();
1793             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1794                 oldFlowConfig = entry;
1795                 index = mapEntry.getKey();
1796                 break;
1797             }
1798         }
1799
1800         if (oldFlowConfig == null) {
1801             String msg = "Attempt to modify a non existing static flow";
1802             String logMsg = msg + ": {}";
1803             log.warn(logMsg, newFlowConfig);
1804             return new Status(StatusCode.NOTFOUND, msg);
1805         }
1806
1807         // Do not attempt to reinstall the flow, warn user
1808         if (newFlowConfig.equals(oldFlowConfig)) {
1809             String msg = "No modification detected";
1810             log.trace("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1811             return new Status(StatusCode.SUCCESS, msg);
1812         }
1813
1814         // If flow is installed, program the network node
1815         status = new Status(StatusCode.SUCCESS, "Saved in config");
1816         if (oldFlowConfig.installInHw()) {
1817             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1818         }
1819
1820         // Update configuration database if programming was successful
1821         if (status.isSuccess()) {
1822             newFlowConfig.setStatus(status.getDescription());
1823             staticFlows.put(index, newFlowConfig);
1824         }
1825
1826         return status;
1827     }
1828
1829     @Override
1830     public Status toggleStaticFlowStatus(String name, Node node) {
1831         return toggleStaticFlowStatus(getStaticFlow(name, node));
1832     }
1833
1834     @Override
1835     public Status toggleStaticFlowStatus(FlowConfig config) {
1836         if (config == null) {
1837             String msg = "Invalid request: null flow config";
1838             log.warn(msg);
1839             return new Status(StatusCode.BADREQUEST, msg);
1840         }
1841         // Validity check for api3 entry point
1842         if (config.isInternalFlow()) {
1843             String msg = "Invalid operation: Controller generated flow cannot be modified";
1844             String logMsg = msg + ": {}";
1845             log.warn(logMsg, config);
1846             return new Status(StatusCode.NOTACCEPTABLE, msg);
1847         }
1848
1849         // Find the config entry
1850         Integer key = 0;
1851         FlowConfig target = null;
1852         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1853             FlowConfig conf = entry.getValue();
1854             if (conf.isByNameAndNodeIdEqual(config)) {
1855                 key = entry.getKey();
1856                 target = conf;
1857                 break;
1858             }
1859         }
1860         if (target != null) {
1861             Status status = target.validate(container);
1862             if (!status.isSuccess()) {
1863                 log.warn(status.getDescription());
1864                 return status;
1865             }
1866             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1867                                     .installFlowEntry(target.getFlowEntry());
1868             if (status.isSuccess()) {
1869                 // Update Configuration database
1870                 target.setStatus(StatusCode.SUCCESS.toString());
1871                 target.toggleInstallation();
1872                 staticFlows.put(key, target);
1873             }
1874             return status;
1875         }
1876
1877         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1878     }
1879
1880     /**
1881      * Reinsert all static flows entries in the cache to force cache updates in
1882      * the cluster. This is useful when only some parameters were changed in the
1883      * entries, like the status.
1884      *
1885      * @param node
1886      *            The node for which the static flow configurations have to be
1887      *            refreshed. If null, all nodes static flows will be refreshed.
1888      */
1889     private void refreshClusterStaticFlowsStatus(Node node) {
1890         // Refresh cluster cache
1891         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1892             if (node == null || entry.getValue().getNode().equals(node)) {
1893                 staticFlows.put(entry.getKey(), entry.getValue());
1894             }
1895         }
1896     }
1897
1898     /**
1899      * Uninstall all the non-internal Flow Entries present in the software view.
1900      * If requested, a copy of each original flow entry will be stored in the
1901      * inactive list so that it can be re-applied when needed (This is typically
1902      * the case when running in the default container and controller moved to
1903      * container mode) NOTE WELL: The routine as long as does a bulk change will
1904      * operate only on the entries for nodes locally attached so to avoid
1905      * redundant operations initiated by multiple nodes
1906      *
1907      * @param preserveFlowEntries
1908      *            if true, a copy of each original entry is stored in the
1909      *            inactive list
1910      */
1911     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1912         log.trace("Uninstalling all non-internal flows");
1913
1914         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1915
1916         // Store entries / create target list
1917         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1918             FlowEntryInstall flowEntries = mapEntry.getValue();
1919             // Skip internal generated static flows
1920             if (!flowEntries.isInternal()) {
1921                 toRemove.add(flowEntries);
1922                 // Store the original entries if requested
1923                 if (preserveFlowEntries) {
1924                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1925                 }
1926             }
1927         }
1928
1929         // Now remove the entries
1930         for (FlowEntryInstall flowEntryHw : toRemove) {
1931             Node n = flowEntryHw.getNode();
1932             if (n != null && connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
1933                 Status status = this.removeEntryInternal(flowEntryHw, false);
1934                 if (!status.isSuccess()) {
1935                     log.trace("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1936                 }
1937             } else {
1938                 log.debug("Not removing entry {} because not connected locally, the remote guy will do it's job",
1939                         flowEntryHw);
1940             }
1941         }
1942     }
1943
1944     /**
1945      * Re-install all the Flow Entries present in the inactive list The inactive
1946      * list will be empty at the end of this call This function is called on the
1947      * default container instance of FRM only when the last container is deleted
1948      */
1949     private void reinstallAllFlowEntries() {
1950         log.trace("Reinstalling all inactive flows");
1951
1952         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1953             this.addEntry(flowEntry, false);
1954         }
1955
1956         // Empty inactive list in any case
1957         inactiveFlows.clear();
1958     }
1959
1960     @Override
1961     public List<FlowConfig> getStaticFlows() {
1962         return new ArrayList<FlowConfig>(staticFlows.values());
1963     }
1964
1965     @Override
1966     public FlowConfig getStaticFlow(String name, Node node) {
1967         ConcurrentMap.Entry<Integer, FlowConfig> entry = getStaticFlowEntry(name, node);
1968         if(entry != null) {
1969             return entry.getValue();
1970         }
1971         return null;
1972     }
1973
1974     @Override
1975     public List<FlowConfig> getStaticFlows(Node node) {
1976         List<FlowConfig> list = new ArrayList<FlowConfig>();
1977         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1978             if (entry.getValue().onNode(node)) {
1979                 list.add(entry.getValue());
1980             }
1981         }
1982         return list;
1983     }
1984
1985     @Override
1986     public List<String> getStaticFlowNamesForNode(Node node) {
1987         List<String> list = new ArrayList<String>();
1988         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1989             if (entry.getValue().onNode(node)) {
1990                 list.add(entry.getValue().getName());
1991             }
1992         }
1993         return list;
1994     }
1995
1996     @Override
1997     public List<Node> getListNodeWithConfiguredFlows() {
1998         Set<Node> set = new HashSet<Node>();
1999         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2000             set.add(entry.getValue().getNode());
2001         }
2002         return new ArrayList<Node>(set);
2003     }
2004
2005     private void loadFlowConfiguration() {
2006         for (ConfigurationObject conf : configurationService.retrieveConfiguration(this, PORT_GROUP_FILE_NAME)) {
2007             addPortGroupConfig(((PortGroupConfig) conf).getName(), ((PortGroupConfig) conf).getMatchString(), true);
2008         }
2009
2010         for (ConfigurationObject conf : configurationService.retrieveConfiguration(this, STATIC_FLOWS_FILE_NAME)) {
2011             addStaticFlowInternal((FlowConfig) conf, true);
2012         }
2013     }
2014
2015     @Override
2016     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2017         return ois.readObject();
2018     }
2019
2020     @Override
2021     public Status saveConfig() {
2022         return saveConfigInternal();
2023     }
2024
2025     private Status saveConfigInternal() {
2026         List<ConfigurationObject> nonDynamicFlows = new ArrayList<ConfigurationObject>();
2027
2028         for (Integer ordinal : staticFlows.keySet()) {
2029             FlowConfig config = staticFlows.get(ordinal);
2030             // Do not save dynamic and controller generated static flows
2031             if (config.isDynamic() || config.isInternalFlow()) {
2032                 continue;
2033             }
2034             nonDynamicFlows.add(config);
2035         }
2036
2037         configurationService.persistConfiguration(nonDynamicFlows, STATIC_FLOWS_FILE_NAME);
2038         configurationService.persistConfiguration(new ArrayList<ConfigurationObject>(portGroupConfigs.values()),
2039                 PORT_GROUP_FILE_NAME);
2040
2041         return new Status(StatusCode.SUCCESS);
2042     }
2043
2044     @Override
2045     public void subnetNotify(Subnet sub, boolean add) {
2046     }
2047
2048     private boolean programInternalFlow(boolean proactive, FlowConfig fc) {
2049         boolean retVal = true; // program flows unless determined otherwise
2050         if(proactive) {
2051             // if the flow already exists do not program
2052             if(flowConfigExists(fc)) {
2053                 retVal = false;
2054             }
2055         } else {
2056             // if the flow does not exist do not program
2057             if(!flowConfigExists(fc)) {
2058                 retVal = false;
2059             }
2060         }
2061         return retVal;
2062     }
2063
2064     /**
2065      * (non-Javadoc)
2066      *
2067      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2068      *      boolean)
2069      *
2070      *      This method can be called from within the OSGi framework context,
2071      *      given the programming operation can take sometime, it not good
2072      *      pratice to have in it's context operations that can take time,
2073      *      hence moving off to a different thread for async processing.
2074      */
2075     private ExecutorService executor;
2076     @Override
2077     public void modeChangeNotify(final Node node, final boolean proactive) {
2078         Callable<Status> modeChangeCallable = new Callable<Status>() {
2079             @Override
2080             public Status call() throws Exception {
2081                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2082
2083                 List<String> puntAction = new ArrayList<String>();
2084                 puntAction.add(ActionType.CONTROLLER.toString());
2085
2086                 FlowConfig allowARP = new FlowConfig();
2087                 allowARP.setInstallInHw(true);
2088                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2089                 allowARP.setPriority("1");
2090                 allowARP.setNode(node);
2091                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2092                         .toUpperCase());
2093                 allowARP.setActions(puntAction);
2094                 defaultConfigs.add(allowARP);
2095
2096                 FlowConfig allowLLDP = new FlowConfig();
2097                 allowLLDP.setInstallInHw(true);
2098                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2099                 allowLLDP.setPriority("1");
2100                 allowLLDP.setNode(node);
2101                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2102                         .toUpperCase());
2103                 allowLLDP.setActions(puntAction);
2104                 defaultConfigs.add(allowLLDP);
2105
2106                 List<String> dropAction = new ArrayList<String>();
2107                 dropAction.add(ActionType.DROP.toString());
2108
2109                 FlowConfig dropAllConfig = new FlowConfig();
2110                 dropAllConfig.setInstallInHw(true);
2111                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2112                         + FlowConfig.INTERNALSTATICFLOWEND);
2113                 dropAllConfig.setPriority("0");
2114                 dropAllConfig.setNode(node);
2115                 dropAllConfig.setActions(dropAction);
2116                 defaultConfigs.add(dropAllConfig);
2117
2118                 log.trace("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2119                 for (FlowConfig fc : defaultConfigs) {
2120                     // check if the frm really needs to act on the notification.
2121                     // this is to check against duplicate notifications
2122                     if(programInternalFlow(proactive, fc)) {
2123                         Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2124                         if (status.isSuccess()) {
2125                             log.trace("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2126                         } else {
2127                             log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2128                                     fc.getName());
2129                         }
2130                     } else {
2131                         log.debug("Got redundant install request for internal flow: {} on node: {}. Request not sent to FRM.", fc.getName(), node);
2132                     }
2133                 }
2134                 return new Status(StatusCode.SUCCESS);
2135             }
2136         };
2137
2138         /*
2139          * Execute the work outside the caller context, this could be an
2140          * expensive operation and we don't want to block the caller for it.
2141          */
2142         this.executor.submit(modeChangeCallable);
2143     }
2144
2145     /**
2146      * Remove from the databases all the flows installed on the node
2147      *
2148      * @param node
2149      */
2150     private void cleanDatabaseForNode(Node node) {
2151         log.trace("Cleaning Flow database for Node {}", node);
2152         if (nodeFlows.containsKey(node)) {
2153             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2154
2155             for (FlowEntryInstall entry : toRemove) {
2156                 updateSwViews(entry, false);
2157             }
2158         }
2159     }
2160
2161     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2162         if (nc == null) {
2163             return false;
2164         }
2165
2166         Match match = flow.getMatch();
2167         if (match.isPresent(MatchType.IN_PORT)) {
2168             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2169             if (matchPort.equals(nc)) {
2170                 return true;
2171             }
2172         }
2173         List<Action> actionsList = flow.getActions();
2174         if (actionsList != null) {
2175             for (Action action : actionsList) {
2176                 if (action instanceof Output) {
2177                     NodeConnector actionPort = ((Output) action).getPort();
2178                     if (actionPort.equals(nc)) {
2179                         return true;
2180                     }
2181                 }
2182             }
2183         }
2184         return false;
2185     }
2186
2187     @Override
2188     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2189         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2190     }
2191
2192     @Override
2193     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2194         boolean updateStaticFlowCluster = false;
2195
2196         switch (type) {
2197         case ADDED:
2198             break;
2199         case CHANGED:
2200             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2201             if (config != null) {
2202                 switch (config.getValue()) {
2203                 case Config.ADMIN_DOWN:
2204                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2205                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2206                     break;
2207                 case Config.ADMIN_UP:
2208                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2209                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2210                     break;
2211                 case Config.ADMIN_UNDEF:
2212                     break;
2213                 default:
2214                 }
2215             }
2216             break;
2217         case REMOVED:
2218             // This is the case where a switch port is removed from the SDN agent space
2219             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2220             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2221             break;
2222         default:
2223
2224         }
2225
2226         if (updateStaticFlowCluster) {
2227             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2228         }
2229     }
2230
2231     /*
2232      * It goes through the static flows configuration, it identifies the ones
2233      * which have the specified node connector as input or output port and
2234      * install them on the network node if they are marked to be installed in
2235      * hardware and their status shows they were not installed yet
2236      */
2237     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2238         boolean updated = false;
2239         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2240         for (FlowConfig flowConfig : flowConfigForNode) {
2241             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2242                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(StatusCode.SUCCESS.toString())) {
2243                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2244                     if (!status.isSuccess()) {
2245                         flowConfig.setStatus(status.getDescription());
2246                     } else {
2247                         flowConfig.setStatus(StatusCode.SUCCESS.toString());
2248                     }
2249                     updated = true;
2250                 }
2251             }
2252         }
2253         return updated;
2254     }
2255
2256     /*
2257      * Remove from the network node all the flows which have the specified node
2258      * connector as input or output port. If any of the flow entry is a static
2259      * flow, it updates the correspondent configuration.
2260      */
2261     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2262         boolean updated = false;
2263         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2264         if (nodeFlowEntries == null) {
2265             return updated;
2266         }
2267         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2268             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2269                 Status status = this.removeEntryInternal(fei, true);
2270                 if (!status.isSuccess()) {
2271                     continue;
2272                 }
2273                 /*
2274                  * If the flow entry is a static flow, then update its
2275                  * configuration
2276                  */
2277                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2278                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2279                     if (flowConfig != null) {
2280                         flowConfig.setStatus(PORT_REMOVED);
2281                         updated = true;
2282                     }
2283                 }
2284             }
2285         }
2286         return updated;
2287     }
2288
2289     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2290         FlowConfig derivedFlow = new FlowConfig(original);
2291         derivedFlow.setDynamic(true);
2292         derivedFlow.setPortGroup(null);
2293         derivedFlow.setName(original.getName() + "_" + configName + "_" + port);
2294         derivedFlow.setIngressPort(port + "");
2295         return derivedFlow;
2296     }
2297
2298     private void addPortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2299         for (FlowConfig staticFlow : staticFlows.values()) {
2300             if (staticFlow.getPortGroup() == null) {
2301                 continue;
2302             }
2303             if ((staticFlow.getNode().equals(node)) && (staticFlow.getPortGroup().equals(config.getName()))) {
2304                 for (Short port : data.getPorts()) {
2305                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2306                     addStaticFlowInternal(derivedFlow, false);
2307                 }
2308             }
2309         }
2310     }
2311
2312     private void removePortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2313         for (FlowConfig staticFlow : staticFlows.values()) {
2314             if (staticFlow.getPortGroup() == null) {
2315                 continue;
2316             }
2317             if (staticFlow.getNode().equals(node) && staticFlow.getPortGroup().equals(config.getName())) {
2318                 for (Short port : data.getPorts()) {
2319                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2320                     removeStaticFlow(derivedFlow);
2321                 }
2322             }
2323         }
2324     }
2325
2326     @Override
2327     public void portGroupChanged(PortGroupConfig config, Map<Node, PortGroup> data, boolean add) {
2328         log.trace("PortGroup Changed for: {} Data: {}", config, portGroupData);
2329         Map<Node, PortGroup> existingData = portGroupData.get(config);
2330         if (existingData != null) {
2331             for (Map.Entry<Node, PortGroup> entry : data.entrySet()) {
2332                 PortGroup existingPortGroup = existingData.get(entry.getKey());
2333                 if (existingPortGroup == null) {
2334                     if (add) {
2335                         existingData.put(entry.getKey(), entry.getValue());
2336                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2337                     }
2338                 } else {
2339                     if (add) {
2340                         existingPortGroup.getPorts().addAll(entry.getValue().getPorts());
2341                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2342                     } else {
2343                         existingPortGroup.getPorts().removeAll(entry.getValue().getPorts());
2344                         removePortGroupFlows(config, entry.getKey(), entry.getValue());
2345                     }
2346                 }
2347             }
2348         } else {
2349             if (add) {
2350                 portGroupData.put(config, data);
2351                 for (Node swid : data.keySet()) {
2352                     addPortGroupFlows(config, swid, data.get(swid));
2353                 }
2354             }
2355         }
2356     }
2357
2358     @Override
2359     public boolean addPortGroupConfig(String name, String regex, boolean restore) {
2360         PortGroupConfig config = portGroupConfigs.get(name);
2361         if (config != null) {
2362             return false;
2363         }
2364
2365         if ((portGroupProvider == null) && !restore) {
2366             return false;
2367         }
2368         if ((portGroupProvider != null) && (!portGroupProvider.isMatchCriteriaSupported(regex))) {
2369             return false;
2370         }
2371
2372         config = new PortGroupConfig(name, regex);
2373         portGroupConfigs.put(name, config);
2374         if (portGroupProvider != null) {
2375             portGroupProvider.createPortGroupConfig(config);
2376         }
2377         return true;
2378     }
2379
2380     @Override
2381     public boolean delPortGroupConfig(String name) {
2382         PortGroupConfig config = portGroupConfigs.get(name);
2383         if (config == null) {
2384             return false;
2385         }
2386
2387         if (portGroupProvider != null) {
2388             portGroupProvider.deletePortGroupConfig(config);
2389         }
2390         portGroupConfigs.remove(name);
2391         return true;
2392     }
2393
2394     @Override
2395     public Map<String, PortGroupConfig> getPortGroupConfigs() {
2396         return portGroupConfigs;
2397     }
2398
2399     public boolean isPortGroupSupported() {
2400         if (portGroupProvider == null) {
2401             return false;
2402         }
2403         return true;
2404     }
2405
2406     public void setIContainer(IContainer s) {
2407         this.container = s;
2408     }
2409
2410     public void unsetIContainer(IContainer s) {
2411         if (this.container == s) {
2412             this.container = null;
2413         }
2414     }
2415
2416     public void setConfigurationContainerService(IConfigurationContainerService service) {
2417         log.trace("Got configuration service set request {}", service);
2418         this.configurationService = service;
2419     }
2420
2421     public void unsetConfigurationContainerService(IConfigurationContainerService service) {
2422         log.trace("Got configuration service UNset request");
2423         this.configurationService = null;
2424     }
2425
2426     @Override
2427     public PortGroupProvider getPortGroupProvider() {
2428         return portGroupProvider;
2429     }
2430
2431     public void unsetPortGroupProvider(PortGroupProvider portGroupProvider) {
2432         this.portGroupProvider = null;
2433     }
2434
2435     public void setPortGroupProvider(PortGroupProvider portGroupProvider) {
2436         this.portGroupProvider = portGroupProvider;
2437         portGroupProvider.registerPortGroupChange(this);
2438         for (PortGroupConfig config : portGroupConfigs.values()) {
2439             portGroupProvider.createPortGroupConfig(config);
2440         }
2441     }
2442
2443     public void setFrmAware(IForwardingRulesManagerAware obj) {
2444         this.frmAware.add(obj);
2445     }
2446
2447     public void unsetFrmAware(IForwardingRulesManagerAware obj) {
2448         this.frmAware.remove(obj);
2449     }
2450
2451     void setClusterContainerService(IClusterContainerServices s) {
2452         log.debug("Cluster Service set");
2453         this.clusterContainerService = s;
2454     }
2455
2456     void unsetClusterContainerService(IClusterContainerServices s) {
2457         if (this.clusterContainerService == s) {
2458             log.debug("Cluster Service removed!");
2459             this.clusterContainerService = null;
2460         }
2461     }
2462
2463     private String getContainerName() {
2464         if (container == null) {
2465             return GlobalConstants.DEFAULT.toString();
2466         }
2467         return container.getName();
2468     }
2469
2470     /**
2471      * Function called by the dependency manager when all the required
2472      * dependencies are satisfied
2473      *
2474      */
2475     void init() {
2476
2477         inContainerMode = false;
2478
2479         if (portGroupProvider != null) {
2480             portGroupProvider.registerPortGroupChange(this);
2481         }
2482
2483         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
2484         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
2485
2486         cacheStartup();
2487
2488         /*
2489          * If we are not the first cluster node to come up, do not initialize
2490          * the static flow entries ordinal
2491          */
2492         if (staticFlowsOrdinal.size() == 0) {
2493             staticFlowsOrdinal.put(0, Integer.valueOf(0));
2494         }
2495
2496         pendingEvents = new LinkedBlockingQueue<FRMEvent>();
2497
2498         // Initialize the event handler thread
2499         frmEventHandler = new Thread(new Runnable() {
2500             @Override
2501             public void run() {
2502                 while (!stopping) {
2503                     try {
2504                         final FRMEvent event = pendingEvents.take();
2505                         if (event == null) {
2506                             log.warn("Dequeued null event");
2507                             continue;
2508                         }
2509                         log.trace("Dequeued {} event", event.getClass().getSimpleName());
2510                         if (event instanceof NodeUpdateEvent) {
2511                             NodeUpdateEvent update = (NodeUpdateEvent) event;
2512                             Node node = update.getNode();
2513                             switch (update.getUpdateType()) {
2514                             case ADDED:
2515                                 addStaticFlowsToSwitch(node);
2516                                 break;
2517                             case REMOVED:
2518                                 cleanDatabaseForNode(node);
2519                                 updateStaticFlowConfigsOnNodeDown(node);
2520                                 break;
2521                             default:
2522                             }
2523                         } else if (event instanceof ErrorReportedEvent) {
2524                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
2525                             processErrorEvent(errEvent);
2526                         } else if (event instanceof WorkOrderEvent) {
2527                             /*
2528                              * Take care of handling the remote Work request
2529                              */
2530                             Runnable r = new Runnable() {
2531                                 @Override
2532                                 public void run() {
2533                                     WorkOrderEvent work = (WorkOrderEvent) event;
2534                                     FlowEntryDistributionOrder fe = work.getFe();
2535                                     if (fe != null) {
2536                                         logsync.trace("Executing the workOrder {}", fe);
2537                                         Status gotStatus = null;
2538                                         FlowEntryInstall feiCurrent = fe.getEntry();
2539                                         FlowEntryInstall feiNew = workOrder.get(fe);
2540                                         switch (fe.getUpType()) {
2541                                         case ADDED:
2542                                             gotStatus = addEntryInHw(feiCurrent, false);
2543                                             break;
2544                                         case CHANGED:
2545                                             gotStatus = modifyEntryInHw(feiCurrent, feiNew, false);
2546                                             break;
2547                                         case REMOVED:
2548                                             gotStatus = removeEntryInHw(feiCurrent, false);
2549                                             break;
2550                                         }
2551                                         // Remove the Order
2552                                         workOrder.remove(fe);
2553                                         logsync.trace(
2554                                                 "The workOrder has been executed and now the status is being returned {}", fe);
2555                                         // Place the status
2556                                         workStatus.put(fe, gotStatus);
2557                                     } else {
2558                                         log.warn("Not expected null WorkOrder", work);
2559                                     }
2560                                 }
2561                             };
2562                             if(executor != null) {
2563                                 executor.execute(r);
2564                             }
2565                         } else if (event instanceof WorkStatusCleanup) {
2566                             /*
2567                              * Take care of handling the remote Work request
2568                              */
2569                             WorkStatusCleanup work = (WorkStatusCleanup) event;
2570                             FlowEntryDistributionOrder fe = work.getFe();
2571                             if (fe != null) {
2572                                 logsync.trace("The workStatus {} is being removed", fe);
2573                                 workStatus.remove(fe);
2574                             } else {
2575                                 log.warn("Not expected null WorkStatus", work);
2576                             }
2577                         }  else if (event instanceof ContainerFlowChangeEvent) {
2578                             /*
2579                              * Whether it is an addition or removal, we have to
2580                              * recompute the merged flows entries taking into
2581                              * account all the current container flows because
2582                              * flow merging is not an injective function
2583                              */
2584                             updateFlowsContainerFlow();
2585                         } else if (event instanceof UpdateIndexDBs) {
2586                             UpdateIndexDBs update = (UpdateIndexDBs)event;
2587                             updateIndexDatabase(update.getFei(), update.isAddition());
2588                         } else {
2589                             log.warn("Dequeued unknown event {}", event.getClass().getSimpleName());
2590                         }
2591                     } catch (InterruptedException e) {
2592                         // clear pending events
2593                         pendingEvents.clear();
2594                     }
2595                 }
2596             }
2597         }, "FRM EventHandler Collector");
2598     }
2599
2600     /**
2601      * Function called by the dependency manager when at least one dependency
2602      * become unsatisfied or when the component is shutting down because for
2603      * example bundle is being stopped.
2604      *
2605      */
2606     void destroy() {
2607         // Interrupt the thread
2608         frmEventHandler.interrupt();
2609         // Clear the pendingEvents queue
2610         pendingEvents.clear();
2611         frmAware.clear();
2612         workMonitor.clear();
2613     }
2614
2615     /**
2616      * Function called by dependency manager after "init ()" is called and after
2617      * the services provided by the class are registered in the service registry
2618      *
2619      */
2620     void start() {
2621         /*
2622          * If running in default container, need to know if controller is in
2623          * container mode
2624          */
2625         if (GlobalConstants.DEFAULT.toString().equals(this.getContainerName())) {
2626             inContainerMode = containerManager.inContainerMode();
2627         }
2628
2629         // Initialize graceful stop flag
2630         stopping = false;
2631
2632         // Allocate the executor service
2633         this.executor = Executors.newFixedThreadPool(maxPoolSize);
2634
2635         // Start event handler thread
2636         frmEventHandler.start();
2637
2638         // replay the installedSwView data structure to populate
2639         // node flows and group flows
2640         for (FlowEntryInstall fei : installedSwView.values()) {
2641             pendingEvents.offer(new UpdateIndexDBs(fei, true));
2642         }
2643
2644         /*
2645          * Read startup and build database if we are the coordinator
2646          */
2647         loadFlowConfiguration();
2648     }
2649
2650     /**
2651      * Function called by the dependency manager before Container is Stopped and Destroyed.
2652      */
2653     public void containerStop() {
2654         uninstallAllFlowEntries(false);
2655     }
2656
2657     /**
2658      * Function called by the dependency manager before the services exported by
2659      * the component are unregistered, this will be followed by a "destroy ()"
2660      * calls
2661      */
2662     void stop() {
2663         stopping = true;
2664         // Shutdown executor
2665         this.executor.shutdownNow();
2666         // Now walk all the workMonitor and wake up the one sleeping because
2667         // destruction is happening
2668         for (FlowEntryDistributionOrder fe : workMonitor.keySet()) {
2669             FlowEntryDistributionOrderFutureTask task = workMonitor.get(fe);
2670             task.cancel(true);
2671         }
2672     }
2673
2674     public void setFlowProgrammerService(IFlowProgrammerService service) {
2675         this.programmer = service;
2676     }
2677
2678     public void unsetFlowProgrammerService(IFlowProgrammerService service) {
2679         if (this.programmer == service) {
2680             this.programmer = null;
2681         }
2682     }
2683
2684     public void setSwitchManager(ISwitchManager switchManager) {
2685         this.switchManager = switchManager;
2686     }
2687
2688     public void unsetSwitchManager(ISwitchManager switchManager) {
2689         if (this.switchManager == switchManager) {
2690             this.switchManager = null;
2691         }
2692     }
2693
2694     @Override
2695     public void tagUpdated(String containerName, Node n, short oldTag, short newTag, UpdateType t) {
2696         if (!container.getName().equals(containerName)) {
2697             return;
2698         }
2699     }
2700
2701     @Override
2702     public void containerFlowUpdated(String containerName, ContainerFlow previous, ContainerFlow current, UpdateType t) {
2703         if (!container.getName().equals(containerName)) {
2704             return;
2705         }
2706         log.trace("Container {}: Updating installed flows because of container flow change: {} {}",
2707                 container.getName(), t, current);
2708         ContainerFlowChangeEvent ev = new ContainerFlowChangeEvent(previous, current, t);
2709         pendingEvents.offer(ev);
2710     }
2711
2712     @Override
2713     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
2714         if (!container.getName().equals(containerName)) {
2715             return;
2716         }
2717
2718         boolean updateStaticFlowCluster = false;
2719
2720         switch (t) {
2721         case REMOVED:
2722             log.trace("Port {} was removed from container: uninstalling interested flows", nc);
2723             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nc);
2724             break;
2725         case ADDED:
2726             log.trace("Port {} was added to container: reinstall interested flows", nc);
2727             updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nc);
2728
2729             break;
2730         case CHANGED:
2731             break;
2732         default:
2733         }
2734
2735         if (updateStaticFlowCluster) {
2736             refreshClusterStaticFlowsStatus(nc.getNode());
2737         }
2738     }
2739
2740     @Override
2741     public void containerModeUpdated(UpdateType update) {
2742         // Only default container instance reacts on this event
2743         if (!container.getName().equals(GlobalConstants.DEFAULT.toString())) {
2744             return;
2745         }
2746         switch (update) {
2747         case ADDED:
2748             /*
2749              * Controller is moving to container mode. We are in the default
2750              * container context, we need to remove all our non-internal flows
2751              * to prevent any container isolation breakage. We also need to
2752              * preserve our flow so that they can be re-installed if we move
2753              * back to non container mode (no containers).
2754              */
2755             this.inContainerMode = true;
2756             this.uninstallAllFlowEntries(true);
2757             break;
2758         case REMOVED:
2759             this.inContainerMode = false;
2760             this.reinstallAllFlowEntries();
2761             break;
2762         default:
2763         }
2764
2765         // Update our configuration DB
2766         updateStaticFlowConfigsOnContainerModeChange(update);
2767     }
2768
2769     protected abstract class FRMEvent {
2770
2771     }
2772
2773     private class NodeUpdateEvent extends FRMEvent {
2774         private final Node node;
2775         private final UpdateType update;
2776
2777         public NodeUpdateEvent(UpdateType update, Node node) {
2778             this.update = update;
2779             this.node = node;
2780         }
2781
2782         public UpdateType getUpdateType() {
2783             return update;
2784         }
2785
2786         public Node getNode() {
2787             return node;
2788         }
2789     }
2790
2791     private class ErrorReportedEvent extends FRMEvent {
2792         private final long rid;
2793         private final Node node;
2794         private final Object error;
2795
2796         public ErrorReportedEvent(long rid, Node node, Object error) {
2797             this.rid = rid;
2798             this.node = node;
2799             this.error = error;
2800         }
2801
2802         public long getRequestId() {
2803             return rid;
2804         }
2805
2806         public Object getError() {
2807             return error;
2808         }
2809
2810         public Node getNode() {
2811             return node;
2812         }
2813     }
2814
2815     private class WorkOrderEvent extends FRMEvent {
2816         private FlowEntryDistributionOrder fe;
2817         private FlowEntryInstall newEntry;
2818
2819         /**
2820          * @param fe
2821          * @param newEntry
2822          */
2823         WorkOrderEvent(FlowEntryDistributionOrder fe, FlowEntryInstall newEntry) {
2824             this.fe = fe;
2825             this.newEntry = newEntry;
2826         }
2827
2828         /**
2829          * @return the fe
2830          */
2831         public FlowEntryDistributionOrder getFe() {
2832             return fe;
2833         }
2834
2835         /**
2836          * @return the newEntry
2837          */
2838         public FlowEntryInstall getNewEntry() {
2839             return newEntry;
2840         }
2841     }
2842     private class ContainerFlowChangeEvent extends FRMEvent {
2843         private final ContainerFlow previous;
2844         private final ContainerFlow current;
2845         private final UpdateType type;
2846
2847         public ContainerFlowChangeEvent(ContainerFlow previous, ContainerFlow current, UpdateType type) {
2848             this.previous = previous;
2849             this.current = current;
2850             this.type = type;
2851         }
2852
2853         public ContainerFlow getPrevious() {
2854             return this.previous;
2855         }
2856
2857         public ContainerFlow getCurrent() {
2858             return this.current;
2859         }
2860
2861         public UpdateType getType() {
2862             return this.type;
2863         }
2864     }
2865
2866
2867     private class WorkStatusCleanup extends FRMEvent {
2868         private FlowEntryDistributionOrder fe;
2869
2870         /**
2871          * @param fe
2872          */
2873         WorkStatusCleanup(FlowEntryDistributionOrder fe) {
2874             this.fe = fe;
2875         }
2876
2877         /**
2878          * @return the fe
2879          */
2880         public FlowEntryDistributionOrder getFe() {
2881             return fe;
2882         }
2883     }
2884
2885     private class UpdateIndexDBs extends FRMEvent {
2886         private FlowEntryInstall fei;
2887         private boolean add;
2888
2889         /**
2890          *
2891          * @param fei the flow entry which was installed/removed on the netwrok node
2892          * @param update
2893          */
2894         UpdateIndexDBs(FlowEntryInstall fei, boolean add) {
2895             this.fei = fei;
2896             this.add = add;
2897         }
2898
2899
2900         /**
2901          * @return the flowEntryInstall object which was added/removed
2902          * to/from the installed software view cache
2903          */
2904         public FlowEntryInstall getFei() {
2905             return fei;
2906         }
2907
2908         /**
2909          *
2910          * @return whether this was an flow addition or removal
2911          */
2912         public boolean isAddition() {
2913             return add;
2914         }
2915     }
2916
2917     @Override
2918     public Status saveConfiguration() {
2919         return saveConfig();
2920     }
2921
2922     @Override
2923     public void flowRemoved(Node node, Flow flow) {
2924         log.trace("Received flow removed notification on {} for {}", node, flow);
2925
2926         // For flow entry identification, only node, match and priority matter
2927         FlowEntryInstall test = new FlowEntryInstall(new FlowEntry("", "", flow, node), null);
2928         FlowEntryInstall installedEntry = this.installedSwView.get(test);
2929         if (installedEntry == null) {
2930             log.trace("Entry is not known to us");
2931             return;
2932         }
2933
2934         // Update Static flow status
2935         Integer key = 0;
2936         FlowConfig target = null;
2937         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2938             FlowConfig conf = entry.getValue();
2939             if (conf.isByNameAndNodeIdEqual(installedEntry.getFlowName(), node)) {
2940                 key = entry.getKey();
2941                 target = conf;
2942                 break;
2943             }
2944         }
2945         if (target != null) {
2946             // Update Configuration database
2947             if (target.getHardTimeout() != null || target.getIdleTimeout() != null) {
2948                 /*
2949                  * No need for checking if actual values: these strings were
2950                  * validated at configuration creation. Also, after a switch
2951                  * down scenario, no use to reinstall a timed flow. Mark it as
2952                  * "do not install". User can manually toggle it.
2953                  */
2954                 target.toggleInstallation();
2955             }
2956             target.setStatus(StatusCode.GONE.toString());
2957             staticFlows.put(key, target);
2958         }
2959
2960         // Update software views
2961         this.updateSwViews(installedEntry, false);
2962     }
2963
2964     @Override
2965     public void flowErrorReported(Node node, long rid, Object err) {
2966         log.trace("Got error {} for message rid {} from node {}", new Object[] { err, rid, node });
2967         pendingEvents.offer(new ErrorReportedEvent(rid, node, err));
2968     }
2969
2970     private void processErrorEvent(ErrorReportedEvent event) {
2971         Node node = event.getNode();
2972         long rid = event.getRequestId();
2973         Object error = event.getError();
2974         String errorString = (error == null) ? "Not provided" : error.toString();
2975         /*
2976          * If this was for a flow install, remove the corresponding entry from
2977          * the software view. If it was a Looking for the rid going through the
2978          * software database. TODO: A more efficient rid <-> FlowEntryInstall
2979          * mapping will have to be added in future
2980          */
2981         FlowEntryInstall target = null;
2982         List<FlowEntryInstall> flowEntryInstallList = nodeFlows.get(node);
2983         // flowEntryInstallList could be null.
2984         // so check for it.
2985         if(flowEntryInstallList != null) {
2986             for (FlowEntryInstall index : flowEntryInstallList) {
2987                 FlowEntryInstall entry = installedSwView.get(index);
2988                 if(entry != null) {
2989                     if (entry.getRequestId() == rid) {
2990                         target = entry;
2991                         break;
2992                     }
2993                 }
2994             }
2995         }
2996         if (target != null) {
2997             // This was a flow install, update database
2998             this.updateSwViews(target, false);
2999             // also update the config
3000             if(FlowConfig.STATICFLOWGROUP.equals(target.getGroupName())) {
3001                 ConcurrentMap.Entry<Integer, FlowConfig> staticFlowEntry = getStaticFlowEntry(target.getFlowName(),target.getNode());
3002                 // staticFlowEntry should never be null.
3003                 // the null check is just an extra defensive check.
3004                 if(staticFlowEntry != null) {
3005                     // Modify status and update cluster cache
3006                     log.debug("Updating static flow configuration on async error event");
3007                     String status = String.format("Cannot be installed on node. reason: %s", errorString);
3008                     staticFlowEntry.getValue().setStatus(status);
3009                     refreshClusterStaticFlowsStatus(node);
3010                 }
3011             }
3012         }
3013
3014         // Notify listeners
3015         if (frmAware != null) {
3016             synchronized (frmAware) {
3017                 for (IForwardingRulesManagerAware frma : frmAware) {
3018                     try {
3019                         frma.requestFailed(rid, errorString);
3020                     } catch (Exception e) {
3021                         log.warn("Failed to notify {}", frma);
3022                     }
3023                 }
3024             }
3025         }
3026     }
3027
3028     @Override
3029     public Status solicitStatusResponse(Node node, boolean blocking) {
3030         Status rv = new Status(StatusCode.INTERNALERROR);
3031
3032         if (this.programmer != null) {
3033             if (blocking) {
3034                 rv = programmer.syncSendBarrierMessage(node);
3035             } else {
3036                 rv = programmer.asyncSendBarrierMessage(node);
3037             }
3038         }
3039
3040         return rv;
3041     }
3042
3043     public void unsetIConnectionManager(IConnectionManager s) {
3044         if (s == this.connectionManager) {
3045             this.connectionManager = null;
3046         }
3047     }
3048
3049     public void setIConnectionManager(IConnectionManager s) {
3050         this.connectionManager = s;
3051     }
3052
3053     public void unsetIContainerManager(IContainerManager s) {
3054         if (s == this.containerManager) {
3055             this.containerManager = null;
3056         }
3057     }
3058
3059     public void setIContainerManager(IContainerManager s) {
3060         this.containerManager = s;
3061     }
3062
3063     @Override
3064     public void entryCreated(Object key, String cacheName, boolean originLocal) {
3065         /*
3066          * Do nothing
3067          */
3068     }
3069
3070     @Override
3071     public void entryUpdated(Object key, Object new_value, String cacheName, boolean originLocal) {
3072         /*
3073          * Streamline the updates for the per node and per group index databases
3074          */
3075         if (cacheName.equals(INSTALLED_SW_VIEW_CACHE)) {
3076             pendingEvents.offer(new UpdateIndexDBs((FlowEntryInstall)new_value, true));
3077         }
3078
3079         if (originLocal) {
3080             /*
3081              * Local updates are of no interest
3082              */
3083             return;
3084         }
3085         if (cacheName.equals(WORK_ORDER_CACHE)) {
3086             logsync.trace("Got a WorkOrderCacheUpdate for {}", key);
3087             /*
3088              * This is the case of one workOrder becoming available, so we need
3089              * to dispatch the work to the appropriate handler
3090              */
3091             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3092             FlowEntryInstall fei = fe.getEntry();
3093             if (fei == null) {
3094                 return;
3095             }
3096             Node n = fei.getNode();
3097             if (connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
3098                 logsync.trace("workOrder for fe {} processed locally", fe);
3099                 // I'm the controller in charge for the request, queue it for
3100                 // processing
3101                 pendingEvents.offer(new WorkOrderEvent(fe, (FlowEntryInstall) new_value));
3102             }
3103         } else if (cacheName.equals(WORK_STATUS_CACHE)) {
3104             logsync.trace("Got a WorkStatusCacheUpdate for {}", key);
3105             /*
3106              * This is the case of one workOrder being completed and a status
3107              * returned
3108              */
3109             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3110             /*
3111              * Check if the order was initiated by this controller in that case
3112              * we need to actually look at the status returned
3113              */
3114             if (fe.getRequestorController()
3115                     .equals(clusterContainerService.getMyAddress())) {
3116                 FlowEntryDistributionOrderFutureTask fet = workMonitor.remove(fe);
3117                 if (fet != null) {
3118                     logsync.trace("workStatus response is for us {}", fe);
3119                     // Signal we got the status
3120                     fet.gotStatus(fe, workStatus.get(fe));
3121                     pendingEvents.offer(new WorkStatusCleanup(fe));
3122                 }
3123             }
3124         }
3125     }
3126
3127     @Override
3128     public void entryDeleted(Object key, String cacheName, boolean originLocal) {
3129         /*
3130          * Streamline the updates for the per node and per group index databases
3131          */
3132         if (cacheName.equals(INSTALLED_SW_VIEW_CACHE)) {
3133             pendingEvents.offer(new UpdateIndexDBs((FlowEntryInstall)key, false));
3134         }
3135     }
3136
3137     /**
3138      * {@inheritDoc}
3139      */
3140     @Override
3141     public List<FlowEntry> getFlowEntriesForNode(Node node) {
3142         List<FlowEntry> list = new ArrayList<FlowEntry>();
3143         if (node != null) {
3144             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
3145                 if (node.equals(entry.getKey().getNode())) {
3146                     list.add(entry.getValue().clone());
3147                 }
3148             }
3149         }
3150         return list;
3151     }
3152
3153     /**
3154      * {@inheritDoc}
3155      */
3156     @Override
3157     public List<FlowEntry> getInstalledFlowEntriesForNode(Node node) {
3158         List<FlowEntry> list = new ArrayList<FlowEntry>();
3159         if (node != null) {
3160             List<FlowEntryInstall> flowEntryInstallList = this.nodeFlows.get(node);
3161             if(flowEntryInstallList != null) {
3162                 for(FlowEntryInstall fi: flowEntryInstallList) {
3163                     list.add(fi.getInstall().clone());
3164                 }
3165             }
3166         }
3167         return list;
3168     }
3169 }