opendaylight/forwardingrulesmanager/implementation/src/main/java/org/opendaylight/controller/forwardingrulesmanager/internal/ForwardingRulesManager.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.forwardingrulesmanager.internal;
  10
  11 import java.io.FileNotFoundException;
  12 import java.io.IOException;
  13 import java.io.ObjectInputStream;
  14 import java.net.UnknownHostException;
  15 import java.util.ArrayList;
  16 import java.util.Collections;
  17 import java.util.EnumSet;
  18 import java.util.HashSet;
  19 import java.util.List;
  20 import java.util.Map;
  21 import java.util.Map.Entry;
  22 import java.util.Set;
  23 import java.util.concurrent.BlockingQueue;
  24 import java.util.concurrent.Callable;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentMap;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.ExecutorService;
  29 import java.util.concurrent.Executors;
  30 import java.util.concurrent.LinkedBlockingQueue;
  31
  32 import org.eclipse.osgi.framework.console.CommandInterpreter;
  33 import org.eclipse.osgi.framework.console.CommandProvider;
  34 import org.opendaylight.controller.clustering.services.CacheConfigException;
  35 import org.opendaylight.controller.clustering.services.CacheExistException;
  36 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
  37 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
  38 import org.opendaylight.controller.clustering.services.IClusterServices;
  39 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
  40 import org.opendaylight.controller.connectionmanager.IConnectionManager;
  41 import org.opendaylight.controller.containermanager.IContainerManager;
  42 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
  43 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
  44 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
  45 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  46 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
  47 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
  48 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
  49 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
  50 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
  51 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
  52 import org.opendaylight.controller.sal.action.Action;
  53 import org.opendaylight.controller.sal.action.ActionType;
  54 import org.opendaylight.controller.sal.action.Output;
  55 import org.opendaylight.controller.sal.connection.ConnectionLocality;
  56 import org.opendaylight.controller.sal.core.Config;
  57 import org.opendaylight.controller.sal.core.ContainerFlow;
  58 import org.opendaylight.controller.sal.core.IContainer;
  59 import org.opendaylight.controller.sal.core.IContainerLocalListener;
  60 import org.opendaylight.controller.sal.core.Node;
  61 import org.opendaylight.controller.sal.core.NodeConnector;
  62 import org.opendaylight.controller.sal.core.Property;
  63 import org.opendaylight.controller.sal.core.UpdateType;
  64 import org.opendaylight.controller.sal.flowprogrammer.Flow;
  65 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
  66 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
  67 import org.opendaylight.controller.sal.match.Match;
  68 import org.opendaylight.controller.sal.match.MatchType;
  69 import org.opendaylight.controller.sal.utils.EtherTypes;
  70 import org.opendaylight.controller.sal.utils.GlobalConstants;
  71 import org.opendaylight.controller.sal.utils.IObjectReader;
  72 import org.opendaylight.controller.sal.utils.NodeCreator;
  73 import org.opendaylight.controller.sal.utils.ObjectReader;
  74 import org.opendaylight.controller.sal.utils.ObjectWriter;
  75 import org.opendaylight.controller.sal.utils.Status;
  76 import org.opendaylight.controller.sal.utils.StatusCode;
  77 import org.opendaylight.controller.switchmanager.IInventoryListener;
  78 import org.opendaylight.controller.switchmanager.ISwitchManager;
  79 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
  80 import org.opendaylight.controller.switchmanager.Subnet;
  81 import org.osgi.framework.BundleContext;
  82 import org.osgi.framework.FrameworkUtil;
  83 import org.slf4j.Logger;
  84 import org.slf4j.LoggerFactory;
  85
  86 /**
  87  * Class that manages forwarding rule installation and removal per container of
  88  * the network. It also maintains the central repository of all the forwarding
  89  * rules installed on the network nodes.
  90  */
  91 public class ForwardingRulesManager implements
  92         IForwardingRulesManager,
  93         PortGroupChangeListener,
  94         IContainerLocalListener,
  95         ISwitchManagerAware,
  96         IConfigurationContainerAware,
  97         IInventoryListener,
  98         IObjectReader,
  99         ICacheUpdateAware<Object,Object>,
 100         CommandProvider,
 101         IFlowProgrammerListener {
 102
 103     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
 104     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
 105     private static final String PORTREMOVED = "Port removed";
 106     private static final String NODEDOWN = "Node is Down";
 107     private static final String INVALID_FLOW_ENTRY = "Invalid FlowEntry";
 108     private String frmFileName;
 109     private String portGroupFileName;
 110     private ConcurrentMap<Integer, FlowConfig> staticFlows;
 111     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
 112     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
 113     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
 114     private ConcurrentMap<String, Object> TSPolicies;
 115     private IContainerManager containerManager;
 116     private boolean inContainerMode; // being used by global instance only
 117     protected boolean stopping;
 118
 119     /*
 120      * Flow database. It's the software view of what was requested to install
 121      * and what is installed on the switch. It is indexed by the entry itself.
 122      * The entry's hashcode resumes the network node index, the flow's priority
 123      * and the flow's match. The value element is a class which contains the
 124      * flow entry pushed by the applications modules and the respective
 125      * container flow merged version. In absence of container flows, the two
 126      * flow entries are the same.
 127      */
 128     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
 129     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
 130     /*
 131      * Per node and per group indexing
 132      */
 133     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
 134     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
 135
 136     /*
 137      * Inactive flow list. This is for the global instance of FRM It will
 138      * contain all the flow entries which were installed on the global container
 139      * when the first container is created.
 140      */
 141     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
 142
 143     private IContainer container;
 144     private Set<IForwardingRulesManagerAware> frmAware =
 145         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
 146     private PortGroupProvider portGroupProvider;
 147     private IFlowProgrammerService programmer;
 148     private IClusterContainerServices clusterContainerService = null;
 149     private ISwitchManager switchManager;
 150     private Thread frmEventHandler;
 151     protected BlockingQueue<FRMEvent> pendingEvents;
 152
 153     // Distributes FRM programming in the cluster
 154     private IConnectionManager connectionManager;
 155
 156     /*
 157      * Name clustered caches used to support FRM entry distribution these are by
 158      * necessity non-transactional as long as need to be able to synchronize
 159      * states also while a transaction is in progress
 160      */
 161     static final String WORK_ORDER_CACHE = "frm.workOrder";
 162     static final String WORK_STATUS_CACHE = "frm.workStatus";
 163
 164     /*
 165      * Data structure responsible for distributing the FlowEntryInstall requests
 166      * in the cluster. The key value is entry that is being either Installed or
 167      * Updated or Delete. The value field is the same of the key value in case
 168      * of Installation or Deletion, it's the new entry in case of Modification,
 169      * this because the clustering caches don't allow null values.
 170      *
 171      * The logic behind this data structure is that the controller that initiate
 172      * the request will place the order here, someone will pick it and then will
 173      * remove from this data structure because is being served.
 174      *
 175      * TODO: We need to have a way to cleanup this data structure if entries are
 176      * not picked by anyone, which is always a case can happen especially on
 177      * Node disconnect cases.
 178      */
 179     protected ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
 180
 181     /*
 182      * Data structure responsible for retrieving the results of the workOrder
 183      * submitted to the cluster.
 184      *
 185      * The logic behind this data structure is that the controller that has
 186      * executed the order will then place the result in workStatus signaling
 187      * that there was a success or a failure.
 188      *
 189      * TODO: The workStatus entries need to have a lifetime associated in case
 190      * of requestor controller leaving the cluster.
 191      */
 192     protected ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
 193
 194     /*
 195      * Local Map used to hold the Future which a caller can use to monitor for
 196      * completion
 197      */
 198     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
 199             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
 200
 201     /*
 202      * Max pool size for the executor
 203      */
 204     private static final int maxPoolSize = 10;
 205
 206     /**
 207      * @param e
 208      *            Entry being installed/updated/removed
 209      * @param u
 210      *            New entry will be placed after the update operation. Valid
 211      *            only for UpdateType.CHANGED, null for all the other cases
 212      * @param t
 213      *            Type of update
 214      * @return a Future object for monitoring the progress of the result, or
 215      *         null in case the processing should take place locally
 216      */
 217     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
 218             UpdateType t) {
 219         // A null entry it's an unexpected condition, anyway it's safe to keep
 220         // the handling local
 221         if (e == null) {
 222             return null;
 223         }
 224
 225         Node n = e.getNode();
 226         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
 227             // Create the work order and distribute it
 228             FlowEntryDistributionOrder fe =
 229                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
 230             // First create the monitor job
 231             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
 232             logsync.trace("Node {} not local so sending fe {}", n, fe);
 233             workMonitor.put(fe, ret);
 234             if (t.equals(UpdateType.CHANGED)) {
 235                 // Then distribute the work
 236                 workOrder.put(fe, u);
 237             } else {
 238                 // Then distribute the work
 239                 workOrder.put(fe, e);
 240             }
 241             logsync.trace("WorkOrder requested");
 242             // Now create an Handle to monitor the execution of the operation
 243             return ret;
 244         }
 245
 246         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
 247         return null;
 248     }
 249
 250     /**
 251      * Adds a flow entry onto the network node It runs various validity checks
 252      * and derive the final container flows merged entries that will be
 253      * attempted to be installed
 254      *
 255      * @param flowEntry
 256      *            the original flow entry application requested to add
 257      * @param async
 258      *            the flag indicating if this is a asynchronous request
 259      * @return the status of this request. In case of asynchronous call, it will
 260      *         contain the unique id assigned to this request
 261      */
 262     private Status addEntry(FlowEntry flowEntry, boolean async) {
 263
 264         // Sanity Check
 265         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 266             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 267             log.warn(logMsg, flowEntry);
 268             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 269         }
 270
 271         /*
 272          * Redundant Check: Check if the request is a redundant one from the
 273          * same application the flowEntry is equal to an existing one. Given we
 274          * do not have an application signature in the requested FlowEntry yet,
 275          * we are here detecting the above condition by comparing the flow
 276          * names, if set. If they are equal to the installed flow, most likely
 277          * this is a redundant installation request from the same application
 278          * and we can silently return success
 279          *
 280          * TODO: in future a sort of application reference list mechanism will
 281          * be added to the FlowEntry so that exact flow can be used by different
 282          * applications.
 283          */
 284         FlowEntry present = this.originalSwView.get(flowEntry);
 285         if (present != null) {
 286             boolean sameFlow = present.getFlow().equals(flowEntry.getFlow());
 287             boolean sameApp = present.getFlowName() != null && present.getFlowName().equals(flowEntry.getFlowName());
 288             if (sameFlow && sameApp) {
 289                 log.trace("Skipping redundant request for flow {} on node {}", flowEntry.getFlowName(),
 290                         flowEntry.getNode());
 291                 return new Status(StatusCode.SUCCESS, "Entry is already present");
 292             }
 293         }
 294
 295         /*
 296          * Derive the container flow merged entries to install In presence of N
 297          * container flows, we may end up with N different entries to install...
 298          */
 299         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 300
 301         // Container Flow conflict Check
 302         if (toInstallList.isEmpty()) {
 303             String msg = "Flow Entry conflicts with all Container Flows";
 304             String logMsg = msg + ": {}";
 305             log.warn(logMsg, flowEntry);
 306             return new Status(StatusCode.CONFLICT, msg);
 307         }
 308
 309         // Derive the list of entries good to be installed
 310         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 311         for (FlowEntryInstall entry : toInstallList) {
 312             // Conflict Check: Verify new entry would not overwrite existing
 313             // ones
 314             if (this.installedSwView.containsKey(entry)) {
 315                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
 316                 log.trace("Aborting to install {}", entry);
 317                 continue;
 318             }
 319             toInstallSafe.add(entry);
 320         }
 321
 322         // Declare failure if all the container flow merged entries clash with
 323         // existing entries
 324         if (toInstallSafe.size() == 0) {
 325             String msg = "A flow with same match and priority exists on the target node";
 326             String logMsg = msg + ": {}";
 327             log.warn(logMsg, flowEntry);
 328             return new Status(StatusCode.CONFLICT, msg);
 329         }
 330
 331         // Try to install an entry at the time
 332         Status error = new Status(null, null);
 333         Status succeded = null;
 334         boolean oneSucceded = false;
 335         for (FlowEntryInstall installEntry : toInstallSafe) {
 336
 337             // Install and update database
 338             Status ret = addEntriesInternal(installEntry, async);
 339
 340             if (ret.isSuccess()) {
 341                 oneSucceded = true;
 342                 /*
 343                  * The first successful status response will be returned For the
 344                  * asynchronous call, we can discard the container flow
 345                  * complication for now and assume we will always deal with one
 346                  * flow only per request
 347                  */
 348                 succeded = ret;
 349             } else {
 350                 error = ret;
 351                 log.trace("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
 352             }
 353         }
 354
 355         return (oneSucceded) ? succeded : error;
 356     }
 357
 358     /**
 359      * Given a flow entry and the list of container flows, it returns the list
 360      * of container flow merged flow entries good to be installed on this
 361      * container. If the list of container flows is null or empty, the install
 362      * entry list will contain only one entry, the original flow entry. If the
 363      * flow entry is congruent with all the N container flows, then the output
 364      * install entry list will contain N entries. If the output list is empty,
 365      * it means the passed flow entry conflicts with all the container flows.
 366      *
 367      * @param cFlowList
 368      *            The list of container flows
 369      * @return the list of container flow merged entries good to be installed on
 370      *         this container
 371      */
 372     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
 373         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
 374
 375         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
 376             // No container flows => entry good to be installed unchanged
 377             toInstallList.add(new FlowEntryInstall(request.clone(), null));
 378         } else {
 379             // Create the list of entries to be installed. If the flow entry is
 380             // not congruent with any container flow, no install entries will be
 381             // created
 382             for (ContainerFlow cFlow : container.getContainerFlows()) {
 383                 if (cFlow.allowsFlow(request.getFlow())) {
 384                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
 385                 }
 386             }
 387         }
 388         return toInstallList;
 389     }
 390
 391     /**
 392      * Modify a flow entry with a new one It runs various validity check and
 393      * derive the final container flows merged flow entries to work with
 394      *
 395      * @param currentFlowEntry
 396      * @param newFlowEntry
 397      * @param async
 398      *            the flag indicating if this is a asynchronous request
 399      * @return the status of this request. In case of asynchronous call, it will
 400      *         contain the unique id assigned to this request
 401      */
 402     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
 403         Status retExt;
 404
 405         // Sanity checks
 406         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
 407                 || newFlowEntry.getNode() == null || newFlowEntry.getFlow() == null) {
 408             String msg = "Modify: " + INVALID_FLOW_ENTRY;
 409             String logMsg = msg + ": {} or {}";
 410             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 411             return new Status(StatusCode.NOTACCEPTABLE, msg);
 412         }
 413         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
 414                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
 415             String msg = "Modify: Incompatible Flow Entries";
 416             String logMsg = msg + ": {} and {}";
 417             log.warn(logMsg, currentFlowEntry, newFlowEntry);
 418             return new Status(StatusCode.NOTACCEPTABLE, msg);
 419         }
 420
 421         // Equality Check
 422         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
 423             String msg = "Modify skipped as flows are the same";
 424             String logMsg = msg + ": {} and {}";
 425             log.debug(logMsg, currentFlowEntry, newFlowEntry);
 426             return new Status(StatusCode.SUCCESS, msg);
 427         }
 428
 429         /*
 430          * Conflict Check: Verify the new entry would not conflict with an
 431          * existing one. This is a loose check on the previous original flow
 432          * entry requests. No check on the container flow merged flow entries
 433          * (if any) yet
 434          */
 435         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
 436         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
 437             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
 438             String logMsg = msg + ": {}";
 439             log.warn(logMsg, currentFlowEntry);
 440             return new Status(StatusCode.CONFLICT, msg);
 441         }
 442
 443         // Derive the installed and toInstall entries
 444         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
 445                 container.getContainerFlows());
 446         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
 447
 448         if (toInstallList.isEmpty()) {
 449             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
 450             String logMsg = msg + ": {}";
 451             log.warn(logMsg, newFlowEntry);
 452             log.warn(msg);
 453             return new Status(StatusCode.CONFLICT, msg);
 454         }
 455
 456         /*
 457          * If the two list sizes differ, it means the new flow entry does not
 458          * satisfy the same number of container flows the current entry does.
 459          * This is only possible when the new entry and current entry have
 460          * different match. In this scenario the modification would ultimately
 461          * be handled as a remove and add operations in the protocol plugin.
 462          *
 463          * Also, if any of the new flow entries would clash with an existing
 464          * one, we cannot proceed with the modify operation, because it would
 465          * fail for some entries and leave stale entries on the network node.
 466          * Modify path can be taken only if it can be performed completely, for
 467          * all entries.
 468          *
 469          * So, for the above two cases, to simplify, let's decouple the modify
 470          * in: 1) remove current entries 2) install new entries
 471          */
 472         Status succeeded = null;
 473         boolean decouple = false;
 474         if (installedList.size() != toInstallList.size()) {
 475             log.info("Modify: New flow entry does not satisfy the same "
 476                     + "number of container flows as the original entry does");
 477             decouple = true;
 478         }
 479         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
 480         for (FlowEntryInstall installEntry : toInstallList) {
 481             /*
 482              * Conflict Check: Verify the new entry would not overwrite another
 483              * existing one
 484              */
 485             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
 486             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
 487                 log.info("Modify: new container flow merged flow entry clashes with existing flow");
 488                 decouple = true;
 489             } else {
 490                 toInstallSafe.add(installEntry);
 491             }
 492         }
 493
 494         if (decouple) {
 495             // Remove current entries
 496             for (FlowEntryInstall currEntry : installedList) {
 497                 this.removeEntryInternal(currEntry, async);
 498             }
 499             // Install new entries
 500             for (FlowEntryInstall newEntry : toInstallSafe) {
 501                 succeeded = this.addEntriesInternal(newEntry, async);
 502             }
 503         } else {
 504             /*
 505              * The two list have the same size and the entries to install do not
 506              * clash with any existing flow on the network node. We assume here
 507              * (and might be wrong) that the same container flows that were
 508              * satisfied by the current entries are the same that are satisfied
 509              * by the new entries. Let's take the risk for now.
 510              *
 511              * Note: modification has to be complete. If any entry modification
 512              * fails, we need to stop, restore the already modified entries, and
 513              * declare failure.
 514              */
 515             Status retModify = null;
 516             int i = 0;
 517             int size = toInstallList.size();
 518             while (i < size) {
 519                 // Modify and update database
 520                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
 521                 if (retModify.isSuccess()) {
 522                     i++;
 523                 } else {
 524                     break;
 525                 }
 526             }
 527             // Check if uncompleted modify
 528             if (i < size) {
 529                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
 530                 // Restore original entries
 531                 int j = 0;
 532                 while (j < i) {
 533                     log.info("Attempting to restore initial entries");
 534                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
 535                     if (retExt.isSuccess()) {
 536                         j++;
 537                     } else {
 538                         break;
 539                     }
 540                 }
 541                 // Fatal error, recovery failed
 542                 if (j < i) {
 543                     String msg = "Flow recovery failed ! Unrecoverable Error";
 544                     log.error(msg);
 545                     return new Status(StatusCode.INTERNALERROR, msg);
 546                 }
 547             }
 548             succeeded = retModify;
 549         }
 550         /*
 551          * The first successful status response will be returned. For the
 552          * asynchronous call, we can discard the container flow complication for
 553          * now and assume we will always deal with one flow only per request
 554          */
 555         return succeeded;
 556     }
 557
 558     /**
 559      * This is the function that modifies the final container flows merged
 560      * entries on the network node and update the database. It expects that all
 561      * the validity checks are passed
 562      *
 563      * @param currentEntries
 564      * @param newEntries
 565      * @param async
 566      *            the flag indicating if this is a asynchronous request
 567      * @return the status of this request. In case of asynchronous call, it will
 568      *         contain the unique id assigned to this request
 569      */
 570     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
 571         FlowEntryDistributionOrderFutureTask futureStatus =
 572                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
 573         if (futureStatus != null) {
 574             Status retStatus = new Status(StatusCode.UNDEFINED);
 575             try {
 576                 retStatus = futureStatus.get();
 577                 if (retStatus.getCode()
 578                         .equals(StatusCode.TIMEOUT)) {
 579                     // A timeout happened, lets cleanup the workMonitor
 580                     workMonitor.remove(futureStatus.getOrder());
 581                 }
 582             } catch (InterruptedException e) {
 583                 log.error("", e);
 584             } catch (ExecutionException e) {
 585                 log.error("", e);
 586             }
 587             return retStatus;
 588         } else {
 589             // Modify the flow on the network node
 590             Status status = async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
 591                     .getFlow(), newEntries.getInstall()
 592                     .getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries.getInstall()
 593                     .getFlow(), newEntries.getInstall()
 594                     .getFlow());
 595
 596             if (!status.isSuccess()) {
 597                 log.trace("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
 598                         status.getDescription());
 599                 return status;
 600             }
 601
 602             log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
 603
 604             // Update DB
 605             newEntries.setRequestId(status.getRequestId());
 606             updateLocalDatabase(currentEntries, false);
 607             updateLocalDatabase(newEntries, true);
 608
 609             return status;
 610         }
 611     }
 612
 613     /**
 614      * Remove a flow entry. If the entry is not present in the software view
 615      * (entry or node not present), it return successfully
 616      *
 617      * @param flowEntry
 618      *            the flow entry to remove
 619      * @param async
 620      *            the flag indicating if this is a asynchronous request
 621      * @return the status of this request. In case of asynchronous call, it will
 622      *         contain the unique id assigned to this request
 623      */
 624     private Status removeEntry(FlowEntry flowEntry, boolean async) {
 625         Status error = new Status(null, null);
 626
 627         // Sanity Check
 628         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
 629             String logMsg = INVALID_FLOW_ENTRY + ": {}";
 630             log.warn(logMsg, flowEntry);
 631             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
 632         }
 633
 634         // Derive the container flows merged installed entries
 635         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
 636
 637         Status succeeded = null;
 638         boolean atLeastOneRemoved = false;
 639         for (FlowEntryInstall entry : installedList) {
 640             if (!installedSwView.containsKey(entry)) {
 641                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
 642                 log.debug(logMsg, flowEntry);
 643                 if (installedList.size() == 1) {
 644                     // If we had only one entry to remove, we are done
 645                     return new Status(StatusCode.SUCCESS);
 646                 } else {
 647                     continue;
 648                 }
 649             }
 650
 651             // Remove and update DB
 652             Status ret = removeEntryInternal(entry, async);
 653
 654             if (!ret.isSuccess()) {
 655                 error = ret;
 656                 log.trace("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
 657                 if (installedList.size() == 1) {
 658                     // If we had only one entry to remove, this is fatal failure
 659                     return error;
 660                 }
 661             } else {
 662                 succeeded = ret;
 663                 atLeastOneRemoved = true;
 664             }
 665         }
 666
 667         /*
 668          * No worries if full removal failed. Consistency checker will take care
 669          * of removing the stale entries later, or adjusting the software
 670          * database if not in sync with hardware
 671          */
 672         return (atLeastOneRemoved) ? succeeded : error;
 673     }
 674
 675     /**
 676      * This is the function that removes the final container flows merged entry
 677      * from the network node and update the database. It expects that all the
 678      * validity checks are passed
 679      *
 680      * @param entry
 681      *            the flow entry to remove
 682      * @param async
 683      *            the flag indicating if this is a asynchronous request
 684      * @return the status of this request. In case of asynchronous call, it will
 685      *         contain the unique id assigned to this request
 686      */
 687     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
 688         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
 689         if (futureStatus != null) {
 690             Status retStatus = new Status(StatusCode.UNDEFINED);
 691             try {
 692                 retStatus = futureStatus.get();
 693                 if (retStatus.getCode()
 694                         .equals(StatusCode.TIMEOUT)) {
 695                     // A timeout happened, lets cleanup the workMonitor
 696                     workMonitor.remove(futureStatus.getOrder());
 697                 }
 698             } catch (InterruptedException e) {
 699                 log.error("", e);
 700             } catch (ExecutionException e) {
 701                 log.error("", e);
 702             }
 703             return retStatus;
 704         } else {
 705             // Mark the entry to be deleted (for CC just in case we fail)
 706             entry.toBeDeleted();
 707
 708             // Remove from node
 709             Status status = async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall()
 710                     .getFlow()) : programmer.removeFlow(entry.getNode(), entry.getInstall()
 711                     .getFlow());
 712
 713             if (!status.isSuccess()) {
 714                 log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", entry.getInstall(),
 715                         status.getDescription());
 716                 return status;
 717             }
 718             log.trace("Removed  {}", entry.getInstall());
 719
 720             // Update DB
 721             updateLocalDatabase(entry, false);
 722
 723             return status;
 724         }
 725     }
 726
 727     /**
 728      * This is the function that installs the final container flow merged entry
 729      * on the network node and updates the database. It expects that all the
 730      * validity and conflict checks are passed. That means it does not check
 731      * whether this flow would conflict or overwrite an existing one.
 732      *
 733      * @param entry
 734      *            the flow entry to install
 735      * @param async
 736      *            the flag indicating if this is a asynchronous request
 737      * @return the status of this request. In case of asynchronous call, it will
 738      *         contain the unique id assigned to this request
 739      */
 740     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
 741         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
 742         if (futureStatus != null) {
 743             Status retStatus = new Status(StatusCode.UNDEFINED);
 744             try {
 745                 retStatus = futureStatus.get();
 746                 if (retStatus.getCode()
 747                         .equals(StatusCode.TIMEOUT)) {
 748                     // A timeout happened, lets cleanup the workMonitor
 749                     workMonitor.remove(futureStatus.getOrder());
 750                 }
 751             } catch (InterruptedException e) {
 752                 log.error("", e);
 753             } catch (ExecutionException e) {
 754                 log.error("", e);
 755             }
 756             return retStatus;
 757         } else {
 758             // Install the flow on the network node
 759             Status status = async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall()
 760                     .getFlow()) : programmer.addFlow(entry.getNode(), entry.getInstall()
 761                     .getFlow());
 762
 763             if (!status.isSuccess()) {
 764                 log.trace("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
 765                         status.getDescription());
 766                 return status;
 767             }
 768
 769             log.trace("Added    {}", entry.getInstall());
 770
 771             // Update DB
 772             entry.setRequestId(status.getRequestId());
 773             updateLocalDatabase(entry, true);
 774
 775             return status;
 776         }
 777     }
 778
 779     /**
 780      * Returns true if the flow conflicts with all the container's flows. This
 781      * means that if the function returns true, the passed flow entry is
 782      * congruent with at least one container flow, hence it is good to be
 783      * installed on this container.
 784      *
 785      * @param flowEntry
 786      * @return true if flow conflicts with all the container flows, false
 787      *         otherwise
 788      */
 789     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
 790         List<ContainerFlow> cFlowList = container.getContainerFlows();
 791
 792         // Validity check and avoid unnecessary computation
 793         // Also takes care of default container where no container flows are
 794         // present
 795         if (cFlowList == null || cFlowList.isEmpty()) {
 796             return false;
 797         }
 798
 799         for (ContainerFlow cFlow : cFlowList) {
 800             if (cFlow.allowsFlow(flowEntry.getFlow())) {
 801                 // Entry is allowed by at least one container flow: good to go
 802                 return false;
 803             }
 804         }
 805         return true;
 806     }
 807
 808     private ConcurrentMap.Entry<Integer, FlowConfig> getStaticFlowEntry(String name, Node node) {
 809         for (ConcurrentMap.Entry<Integer, FlowConfig> flowEntry : staticFlows.entrySet()) {
 810             FlowConfig flowConfig = flowEntry.getValue();
 811             if (flowConfig.isByNameAndNodeIdEqual(name, node)) {
 812                 return flowEntry;
 813             }
 814         }
 815         return null;
 816     }
 817
 818     private void updateLocalDatabase(FlowEntryInstall entry, boolean add) {
 819         // Update the software view
 820         updateSwViewes(entry, add);
 821
 822         // Update node indexed flow database
 823         updateNodeFlowsDB(entry, add);
 824
 825         // Update group indexed flow database
 826         updateGroupFlowsDB(entry, add);
 827     }
 828
 829     /*
 830      * Update the node mapped flows database
 831      */
 832     private void updateSwViewes(FlowEntryInstall flowEntries, boolean add) {
 833         if (add) {
 834             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
 835             installedSwView.put(flowEntries, flowEntries);
 836         } else {
 837             originalSwView.remove(flowEntries.getOriginal());
 838             installedSwView.remove(flowEntries);
 839         }
 840     }
 841
 842     /*
 843      * Update the node mapped flows database
 844      */
 845     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 846         Node node = flowEntries.getNode();
 847
 848         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
 849         if (nodeIndeces == null) {
 850             if (!add) {
 851                 return;
 852             } else {
 853                 nodeIndeces = new ArrayList<FlowEntryInstall>();
 854             }
 855         }
 856
 857         if (add) {
 858             nodeIndeces.add(flowEntries);
 859         } else {
 860             nodeIndeces.remove(flowEntries);
 861         }
 862
 863         // Update cache across cluster
 864         if (nodeIndeces.isEmpty()) {
 865             this.nodeFlows.remove(node);
 866         } else {
 867             this.nodeFlows.put(node, nodeIndeces);
 868         }
 869     }
 870
 871     /*
 872      * Update the group name mapped flows database
 873      */
 874     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
 875         String groupName = flowEntries.getGroupName();
 876
 877         // Flow may not be part of a group
 878         if (groupName == null) {
 879             return;
 880         }
 881
 882         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
 883         if (indices == null) {
 884             if (!add) {
 885                 return;
 886             } else {
 887                 indices = new ArrayList<FlowEntryInstall>();
 888             }
 889         }
 890
 891         if (add) {
 892             indices.add(flowEntries);
 893         } else {
 894             indices.remove(flowEntries);
 895         }
 896
 897         // Update cache across cluster
 898         if (indices.isEmpty()) {
 899             this.groupFlows.remove(groupName);
 900         } else {
 901             this.groupFlows.put(groupName, indices);
 902         }
 903     }
 904
 905     /**
 906      * Remove a flow entry that has been added previously First checks if the
 907      * entry is effectively present in the local database
 908      */
 909     @SuppressWarnings("unused")
 910     private Status removeEntry(Node node, String flowName) {
 911         FlowEntryInstall target = null;
 912
 913         // Find in database
 914         for (FlowEntryInstall entry : installedSwView.values()) {
 915             if (entry.equalsByNodeAndName(node, flowName)) {
 916                 target = entry;
 917                 break;
 918             }
 919         }
 920
 921         // If it is not there, stop any further processing
 922         if (target == null) {
 923             return new Status(StatusCode.SUCCESS, "Entry is not present");
 924         }
 925
 926         // Remove from node
 927         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
 928
 929         // Update DB
 930         if (status.isSuccess()) {
 931             updateLocalDatabase(target, false);
 932         } else {
 933             // log the error
 934             log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
 935                     status.getDescription());
 936         }
 937
 938         return status;
 939     }
 940
 941     @Override
 942     public Status installFlowEntry(FlowEntry flowEntry) {
 943         Status status;
 944         if (isContainerModeAllowed(flowEntry)) {
 945             status = addEntry(flowEntry, false);
 946         } else {
 947             String msg = "Controller in container mode: Install Refused";
 948             String logMsg = msg + ": {}";
 949             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 950             log.warn(logMsg, flowEntry);
 951         }
 952         return status;
 953     }
 954
 955     @Override
 956     public Status installFlowEntryAsync(FlowEntry flowEntry) {
 957         Status status;
 958         if (isContainerModeAllowed(flowEntry)) {
 959             status = addEntry(flowEntry, true);
 960         } else {
 961             String msg = "Controller in container mode: Install Refused";
 962             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 963             log.warn(msg);
 964         }
 965         return status;
 966     }
 967
 968     @Override
 969     public Status uninstallFlowEntry(FlowEntry flowEntry) {
 970         Status status;
 971         if (isContainerModeAllowed(flowEntry)) {
 972             status = removeEntry(flowEntry, false);
 973         } else {
 974             String msg = "Controller in container mode: Uninstall Refused";
 975             String logMsg = msg + ": {}";
 976             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 977             log.warn(logMsg, flowEntry);
 978         }
 979         return status;
 980     }
 981
 982     @Override
 983     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
 984         Status status;
 985         if (isContainerModeAllowed(flowEntry)) {
 986             status = removeEntry(flowEntry, true);
 987         } else {
 988             String msg = "Controller in container mode: Uninstall Refused";
 989             status = new Status(StatusCode.NOTACCEPTABLE, msg);
 990             log.warn(msg);
 991         }
 992         return status;
 993     }
 994
 995     @Override
 996     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
 997         Status status = null;
 998         if (isContainerModeAllowed(currentFlowEntry)) {
 999             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1000         } else {
1001             String msg = "Controller in container mode: Modify Refused";
1002             String logMsg = msg + ": {}";
1003             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1004             log.warn(logMsg, newFlowEntry);
1005         }
1006         return status;
1007     }
1008
1009     @Override
1010     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1011         Status status = null;
1012         if (isContainerModeAllowed(currentFlowEntry)) {
1013             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
1014         } else {
1015             String msg = "Controller in container mode: Modify Refused";
1016             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1017             log.warn(msg);
1018         }
1019         return status;
1020     }
1021
1022     /**
1023      * Returns whether the specified flow entry is allowed to be
1024      * installed/removed/modified based on the current container mode status.
1025      * This call always returns true in the container instance of forwarding
1026      * rules manager. It is meant for the global instance only (default
1027      * container) of forwarding rules manager. Idea is that for assuring
1028      * container isolation of traffic, flow installation in default container is
1029      * blocked when in container mode (containers are present). The only flows
1030      * that are allowed in container mode in the default container are the
1031      * proactive flows, the ones automatically installed on the network node
1032      * which forwarding mode has been configured to "proactive". These flows are
1033      * needed by controller to discover the nodes topology and to discover the
1034      * attached hosts for some SDN switches.
1035      *
1036      * @param flowEntry
1037      *            The flow entry to be installed/removed/modified
1038      * @return true if not in container mode or if flowEntry is internally
1039      *         generated
1040      */
1041     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1042         return (!inContainerMode) ? true : flowEntry.isInternal();
1043     }
1044
1045     @Override
1046     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1047         /*
1048          * Run a check on the original entries to decide whether to go with a
1049          * add or modify method. A loose check means only check against the
1050          * original flow entry requests and not against the installed flow
1051          * entries which are the result of the original entry merged with the
1052          * container flow(s) (if any). The modifyFlowEntry method in presence of
1053          * conflicts with the Container flows (if any) would revert back to a
1054          * delete + add pattern
1055          */
1056         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1057
1058         if (currentFlowEntry != null) {
1059             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1060         } else {
1061             return installFlowEntry(newFlowEntry);
1062         }
1063     }
1064
1065     @Override
1066     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1067         /*
1068          * Run a check on the original entries to decide whether to go with a
1069          * add or modify method. A loose check means only check against the
1070          * original flow entry requests and not against the installed flow
1071          * entries which are the result of the original entry merged with the
1072          * container flow(s) (if any). The modifyFlowEntry method in presence of
1073          * conflicts with the Container flows (if any) would revert back to a
1074          * delete + add pattern
1075          */
1076         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1077
1078         if (currentFlowEntry != null) {
1079             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1080         } else {
1081             return installFlowEntryAsync(newFlowEntry);
1082         }
1083     }
1084
1085     @Override
1086     public Status uninstallFlowEntryGroup(String groupName) {
1087         if (groupName == null || groupName.isEmpty()) {
1088             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1089         }
1090         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1091             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1092         }
1093         if (inContainerMode) {
1094             String msg = "Controller in container mode: Group Uninstall Refused";
1095             String logMsg = msg + ": {}";
1096             log.warn(logMsg, groupName);
1097             return new Status(StatusCode.NOTACCEPTABLE, msg);
1098         }
1099         int toBeRemoved = 0;
1100         String error = "";
1101         if (groupFlows.containsKey(groupName)) {
1102             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1103             toBeRemoved = list.size();
1104             for (FlowEntryInstall entry : list) {
1105                 Status status = this.removeEntry(entry.getOriginal(), false);
1106                 if (status.isSuccess()) {
1107                     toBeRemoved -= 1;
1108                 } else {
1109                     error = status.getDescription();
1110                 }
1111             }
1112         }
1113         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1114                 "Not all the flows were removed: " + error);
1115     }
1116
1117     @Override
1118     public Status uninstallFlowEntryGroupAsync(String groupName) {
1119         if (groupName == null || groupName.isEmpty()) {
1120             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1121         }
1122         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1123             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1124         }
1125         if (inContainerMode) {
1126             String msg = "Controller in container mode: Group Uninstall Refused";
1127             String logMsg = msg + ": {}";
1128             log.warn(logMsg, groupName);
1129             return new Status(StatusCode.NOTACCEPTABLE, msg);
1130         }
1131         if (groupFlows.containsKey(groupName)) {
1132             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1133             for (FlowEntryInstall entry : list) {
1134                 this.removeEntry(entry.getOriginal(), true);
1135             }
1136         }
1137         return new Status(StatusCode.SUCCESS);
1138     }
1139
1140     @Override
1141     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1142         return entryConflictsWithContainerFlows(flowEntry);
1143     }
1144
1145     /**
1146      * Updates all installed flows because the container flow got updated This
1147      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1148      * Reinstall of all flows This is needed because a new container flows
1149      * merged flow may conflict with an existing old container flows merged flow
1150      * on the network node
1151      */
1152     protected void updateFlowsContainerFlow() {
1153         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1154         // First remove all installed entries
1155         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1156             FlowEntryInstall current = entry.getValue();
1157             // Store the original entry
1158             toReInstall.add(current.getOriginal());
1159             // Remove the old couples. No validity checks to be run, use the
1160             // internal remove
1161             this.removeEntryInternal(current, false);
1162         }
1163         // Then reinstall the original entries
1164         for (FlowEntry entry : toReInstall) {
1165             // Reinstall the original flow entries, via the regular path: new
1166             // cFlow merge + validations
1167             this.installFlowEntry(entry);
1168         }
1169     }
1170
1171     private void nonClusterObjectCreate() {
1172         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1173         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1174         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
1175         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
1176         TSPolicies = new ConcurrentHashMap<String, Object>();
1177         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1178         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1179         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1180         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1181         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1182     }
1183
1184     private void registerWithOSGIConsole() {
1185         BundleContext bundleContext = FrameworkUtil.getBundle(this.getClass()).getBundleContext();
1186         bundleContext.registerService(CommandProvider.class.getName(), this, null);
1187     }
1188
1189     @Override
1190     public void setTSPolicyData(String policyname, Object o, boolean add) {
1191
1192         if (add) {
1193             /* Check if this policy already exists */
1194             if (!(TSPolicies.containsKey(policyname))) {
1195                 TSPolicies.put(policyname, o);
1196             }
1197         } else {
1198             TSPolicies.remove(policyname);
1199         }
1200         if (frmAware != null) {
1201             synchronized (frmAware) {
1202                 for (IForwardingRulesManagerAware frma : frmAware) {
1203                     try {
1204                         frma.policyUpdate(policyname, add);
1205                     } catch (Exception e) {
1206                         log.warn("Exception on callback", e);
1207                     }
1208                 }
1209             }
1210         }
1211     }
1212
1213     @Override
1214     public Map<String, Object> getTSPolicyData() {
1215         return TSPolicies;
1216     }
1217
1218     @Override
1219     public Object getTSPolicyData(String policyName) {
1220         if (TSPolicies.containsKey(policyName)) {
1221             return TSPolicies.get(policyName);
1222         } else {
1223             return null;
1224         }
1225     }
1226
1227     @Override
1228     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1229         List<FlowEntry> list = new ArrayList<FlowEntry>();
1230         if (policyName != null && !policyName.trim().isEmpty()) {
1231             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1232                 if (policyName.equals(entry.getKey().getGroupName())) {
1233                     list.add(entry.getKey().clone());
1234                 }
1235             }
1236         }
1237         return list;
1238     }
1239
1240     @Override
1241     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1242         List<FlowEntry> list = new ArrayList<FlowEntry>();
1243         if (policyName != null && !policyName.trim().isEmpty()) {
1244             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1245                 if (policyName.equals(entry.getKey().getGroupName())) {
1246                     list.add(entry.getKey().getInstall().clone());
1247                 }
1248             }
1249         }
1250         return list;
1251     }
1252
1253     @Override
1254     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1255
1256         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1257             if (flow.getFlowName().equals(flowName)) {
1258                 FlowEntry currentFlowEntry = flow.getOriginal();
1259                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1260                 for (NodeConnector dstPort : portList) {
1261                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1262                 }
1263                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1264                 if (error.isSuccess()) {
1265                     log.info("Ports {} added to FlowEntry {}", portList, flowName);
1266                 } else {
1267                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1268                             currentFlowEntry.toString(), error.getDescription());
1269                 }
1270                 return;
1271             }
1272         }
1273         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1274     }
1275
1276     @Override
1277     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1278         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1279             FlowEntryInstall flow = this.installedSwView.get(index);
1280             if (flow.getFlowName().equals(flowName)) {
1281                 FlowEntry currentFlowEntry = flow.getOriginal();
1282                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1283                 for (NodeConnector dstPort : portList) {
1284                     Action action = new Output(dstPort);
1285                     newFlowEntry.getFlow().removeAction(action);
1286                 }
1287                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1288                 if (status.isSuccess()) {
1289                     log.info("Ports {} removed from FlowEntry {}", portList, flowName);
1290                 } else {
1291                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1292                             currentFlowEntry.toString(), status.getDescription());
1293                 }
1294                 return;
1295             }
1296         }
1297         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1298     }
1299
1300     /*
1301      * This function assumes the target flow has only one output port
1302      */
1303     @Override
1304     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1305         FlowEntry currentFlowEntry = null;
1306         FlowEntry newFlowEntry = null;
1307
1308         // Find the flow
1309         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1310             FlowEntryInstall flow = this.installedSwView.get(index);
1311             if (flow.getFlowName().equals(flowName)) {
1312                 currentFlowEntry = flow.getOriginal();
1313                 break;
1314             }
1315         }
1316         if (currentFlowEntry == null) {
1317             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1318             return;
1319         }
1320
1321         // Create a flow copy with the new output port
1322         newFlowEntry = currentFlowEntry.clone();
1323         Action target = null;
1324         for (Action action : newFlowEntry.getFlow().getActions()) {
1325             if (action.getType() == ActionType.OUTPUT) {
1326                 target = action;
1327                 break;
1328             }
1329         }
1330         newFlowEntry.getFlow().removeAction(target);
1331         newFlowEntry.getFlow().addAction(new Output(outPort));
1332
1333         // Modify on network node
1334         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1335
1336         if (status.isSuccess()) {
1337             log.info("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1338         } else {
1339             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1340                     status.getDescription());
1341         }
1342         return;
1343     }
1344
1345     @Override
1346     public NodeConnector getOutputPort(Node node, String flowName) {
1347         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1348             FlowEntryInstall flow = this.installedSwView.get(index);
1349             if (flow.getFlowName().equals(flowName)) {
1350                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1351                     if (action.getType() == ActionType.OUTPUT) {
1352                         return ((Output) action).getPort();
1353                     }
1354                 }
1355             }
1356         }
1357         return null;
1358     }
1359
1360     private void cacheStartup() {
1361         allocateCaches();
1362         retrieveCaches();
1363     }
1364
1365     private void allocateCaches() {
1366         if (this.clusterContainerService == null) {
1367             log.warn("Un-initialized clusterContainerService, can't create cache");
1368             return;
1369         }
1370
1371         log.debug("Allocating caches for Container {}", container.getName());
1372
1373         try {
1374             clusterContainerService.createCache("frm.originalSwView",
1375                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1376
1377             clusterContainerService.createCache("frm.installedSwView",
1378                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1379
1380             clusterContainerService.createCache("frm.inactiveFlows",
1381                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1382
1383             clusterContainerService.createCache("frm.nodeFlows",
1384                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1385
1386             clusterContainerService.createCache("frm.groupFlows",
1387                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1388
1389             clusterContainerService.createCache("frm.staticFlows",
1390                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1391
1392             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1393                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1394
1395             clusterContainerService.createCache("frm.portGroupConfigs",
1396                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1397
1398             clusterContainerService.createCache("frm.portGroupData",
1399                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1400
1401             clusterContainerService.createCache("frm.TSPolicies",
1402                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1403
1404             clusterContainerService.createCache(WORK_STATUS_CACHE,
1405                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1406
1407             clusterContainerService.createCache(WORK_ORDER_CACHE,
1408                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1409
1410         } catch (CacheConfigException cce) {
1411             log.error("CacheConfigException");
1412         } catch (CacheExistException cce) {
1413             log.error("CacheExistException");
1414         }
1415     }
1416
1417     @SuppressWarnings({ "unchecked" })
1418     private void retrieveCaches() {
1419         ConcurrentMap<?, ?> map;
1420
1421         if (this.clusterContainerService == null) {
1422             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1423             nonClusterObjectCreate();
1424             return;
1425         }
1426
1427         log.debug("Retrieving Caches for Container {}", container.getName());
1428
1429         map = clusterContainerService.getCache("frm.originalSwView");
1430         if (map != null) {
1431             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1432         } else {
1433             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1434         }
1435
1436         map = clusterContainerService.getCache("frm.installedSwView");
1437         if (map != null) {
1438             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1439         } else {
1440             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1441         }
1442
1443         map = clusterContainerService.getCache("frm.inactiveFlows");
1444         if (map != null) {
1445             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1446         } else {
1447             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1448         }
1449
1450         map = clusterContainerService.getCache("frm.nodeFlows");
1451         if (map != null) {
1452             nodeFlows = (ConcurrentMap<Node, List<FlowEntryInstall>>) map;
1453         } else {
1454             log.error("Retrieval of cache failed for Container {}", container.getName());
1455         }
1456
1457         map = clusterContainerService.getCache("frm.groupFlows");
1458         if (map != null) {
1459             groupFlows = (ConcurrentMap<String, List<FlowEntryInstall>>) map;
1460         } else {
1461             log.error("Retrieval of frm.groupFlows cache failed for Container {}", container.getName());
1462         }
1463
1464         map = clusterContainerService.getCache("frm.staticFlows");
1465         if (map != null) {
1466             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1467         } else {
1468             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1469         }
1470
1471         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1472         if (map != null) {
1473             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1474         } else {
1475             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1476         }
1477
1478         map = clusterContainerService.getCache("frm.portGroupConfigs");
1479         if (map != null) {
1480             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1481         } else {
1482             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1483         }
1484
1485         map = clusterContainerService.getCache("frm.portGroupData");
1486         if (map != null) {
1487             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1488         } else {
1489             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1490         }
1491
1492         map = clusterContainerService.getCache("frm.TSPolicies");
1493         if (map != null) {
1494             TSPolicies = (ConcurrentMap<String, Object>) map;
1495         } else {
1496             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1497         }
1498
1499         map = clusterContainerService.getCache(WORK_ORDER_CACHE);
1500         if (map != null) {
1501             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1502         } else {
1503             log.error("Retrieval of " + WORK_ORDER_CACHE + " cache failed for Container {}", container.getName());
1504         }
1505
1506         map = clusterContainerService.getCache(WORK_STATUS_CACHE);
1507         if (map != null) {
1508             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1509         } else {
1510             log.error("Retrieval of " + WORK_STATUS_CACHE + " cache failed for Container {}", container.getName());
1511         }
1512     }
1513
1514     private boolean flowConfigExists(FlowConfig config) {
1515         // Flow name has to be unique on per node id basis
1516         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1517             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1518                 return true;
1519             }
1520         }
1521         return false;
1522     }
1523
1524     @Override
1525     public Status addStaticFlow(FlowConfig config) {
1526         // Configuration object validation
1527         Status status = config.validate(container);
1528         if (!status.isSuccess()) {
1529             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1530             String error = "Invalid Configuration (" + status.getDescription() + ")";
1531             config.setStatus(error);
1532             return new Status(StatusCode.BADREQUEST, error);
1533         }
1534         return addStaticFlowInternal(config, false);
1535     }
1536
1537     /**
1538      * Private method to add a static flow configuration which does not run any
1539      * validation on the passed FlowConfig object. If restore is set to true,
1540      * configuration is stored in configuration database regardless the
1541      * installation on the network node was successful. This is useful at boot
1542      * when static flows are present in startup configuration and are read
1543      * before the switches connects.
1544      *
1545      * @param config
1546      *            The static flow configuration
1547      * @param restore
1548      *            if true, the configuration is stored regardless the
1549      *            installation on the network node was successful
1550      * @return The status of this request
1551      */
1552     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1553         boolean multipleFlowPush = false;
1554         String error;
1555         Status status;
1556         config.setStatus(StatusCode.SUCCESS.toString());
1557
1558         // Presence check
1559         if (flowConfigExists(config)) {
1560             error = "Entry with this name on specified switch already exists";
1561             log.warn("Entry with this name on specified switch already exists: {}", config);
1562             config.setStatus(error);
1563             return new Status(StatusCode.CONFLICT, error);
1564         }
1565
1566         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1567             for (String portGroupName : portGroupConfigs.keySet()) {
1568                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1569                     multipleFlowPush = true;
1570                     break;
1571                 }
1572             }
1573             if (!multipleFlowPush) {
1574                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1575                 error = "Invalid Configuration (Invalid PortGroup Name)";
1576                 config.setStatus(error);
1577                 return new Status(StatusCode.BADREQUEST, error);
1578             }
1579         }
1580
1581         /*
1582          * If requested program the entry in hardware first before updating the
1583          * StaticFlow DB
1584          */
1585         if (!multipleFlowPush) {
1586             // Program hw
1587             if (config.installInHw()) {
1588                 FlowEntry entry = config.getFlowEntry();
1589                 status = this.installFlowEntry(entry);
1590                 if (!status.isSuccess()) {
1591                     config.setStatus(status.getDescription());
1592                     if (!restore) {
1593                         return status;
1594                     }
1595                 }
1596             }
1597         }
1598
1599         /*
1600          * When the control reaches this point, either of the following
1601          * conditions is true 1. This is a single entry configuration (non
1602          * PortGroup) and the hardware installation is successful 2. This is a
1603          * multiple entry configuration (PortGroup) and hardware installation is
1604          * NOT done directly on this event. 3. The User prefers to retain the
1605          * configuration in Controller and skip hardware installation.
1606          *
1607          * Hence it is safe to update the StaticFlow DB at this point.
1608          *
1609          * Note : For the case of PortGrouping, it is essential to have this DB
1610          * populated before the PortGroupListeners can query for the DB
1611          * triggered using portGroupChanged event...
1612          */
1613         Integer ordinal = staticFlowsOrdinal.get(0);
1614         staticFlowsOrdinal.put(0, ++ordinal);
1615         staticFlows.put(ordinal, config);
1616
1617         if (multipleFlowPush) {
1618             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1619             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1620             if (existingData != null) {
1621                 portGroupChanged(pgconfig, existingData, true);
1622             }
1623         }
1624         return new Status(StatusCode.SUCCESS);
1625     }
1626
1627     private void addStaticFlowsToSwitch(Node node) {
1628         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1629             FlowConfig config = entry.getValue();
1630             if (config.isPortGroupEnabled()) {
1631                 continue;
1632             }
1633             if (config.getNode().equals(node)) {
1634                 if (config.installInHw() && !config.getStatus().equals(StatusCode.SUCCESS.toString())) {
1635                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1636                     config.setStatus(status.getDescription());
1637                 }
1638             }
1639         }
1640         // Update cluster cache
1641         refreshClusterStaticFlowsStatus(node);
1642     }
1643
1644     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1645         log.trace("Updating Static Flow configs on node down: {}", node);
1646
1647         List<Integer> toRemove = new ArrayList<Integer>();
1648         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1649
1650             FlowConfig config = entry.getValue();
1651
1652             if (config.isPortGroupEnabled()) {
1653                 continue;
1654             }
1655
1656             if (config.installInHw() && config.getNode().equals(node)) {
1657                 if (config.isInternalFlow()) {
1658                     // Take note of this controller generated static flow
1659                     toRemove.add(entry.getKey());
1660                 } else {
1661                     config.setStatus(NODEDOWN);
1662                 }
1663             }
1664         }
1665         // Remove controller generated static flows for this node
1666         for (Integer index : toRemove) {
1667             staticFlows.remove(index);
1668         }
1669         // Update cluster cache
1670         refreshClusterStaticFlowsStatus(node);
1671
1672     }
1673
1674     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1675         log.trace("Updating Static Flow configs on container mode change: {}", update);
1676
1677         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1678             FlowConfig config = entry.getValue();
1679             if (config.isPortGroupEnabled()) {
1680                 continue;
1681             }
1682             if (config.installInHw() && !config.isInternalFlow()) {
1683                 switch (update) {
1684                 case ADDED:
1685                     config.setStatus("Removed from node because in container mode");
1686                     break;
1687                 case REMOVED:
1688                     config.setStatus(StatusCode.SUCCESS.toString());
1689                     break;
1690                 default:
1691                 }
1692             }
1693         }
1694         // Update cluster cache
1695         refreshClusterStaticFlowsStatus(null);
1696     }
1697
1698     @Override
1699     public Status removeStaticFlow(FlowConfig config) {
1700         /*
1701          * No config.isInternal() check as NB does not take this path and GUI
1702          * cannot issue a delete on an internal generated flow. We need this
1703          * path to be accessible when switch mode is changed from proactive to
1704          * reactive, so that we can remove the internal generated LLDP and ARP
1705          * punt flows
1706          */
1707
1708         // Look for the target configuration entry
1709         Integer key = 0;
1710         FlowConfig target = null;
1711         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1712             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1713                 key = entry.getKey();
1714                 target = entry.getValue();
1715                 break;
1716             }
1717         }
1718         if (target == null) {
1719             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1720         }
1721
1722         // Program the network node
1723         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1724
1725         // Update configuration database if programming was successful
1726         if (status.isSuccess()) {
1727             staticFlows.remove(key);
1728         }
1729
1730         return status;
1731     }
1732
1733     @Override
1734     public Status removeStaticFlow(String name, Node node) {
1735         // Look for the target configuration entry
1736         Integer key = 0;
1737         FlowConfig target = null;
1738         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1739             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1740                 key = mapEntry.getKey();
1741                 target = mapEntry.getValue();
1742                 break;
1743             }
1744         }
1745         if (target == null) {
1746             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1747         }
1748
1749         // Validity check for api3 entry point
1750         if (target.isInternalFlow()) {
1751             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1752             String logMsg = msg + ": {}";
1753             log.warn(logMsg, name);
1754             return new Status(StatusCode.NOTACCEPTABLE, msg);
1755         }
1756
1757         if (target.isPortGroupEnabled()) {
1758             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1759             String logMsg = msg + ": {}";
1760             log.warn(logMsg, name);
1761             return new Status(StatusCode.NOTACCEPTABLE, msg);
1762         }
1763
1764         // Program the network node
1765         Status status = this.removeEntry(target.getFlowEntry(), false);
1766
1767         // Update configuration database if programming was successful
1768         if (status.isSuccess()) {
1769             staticFlows.remove(key);
1770         }
1771
1772         return status;
1773     }
1774
1775     @Override
1776     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1777         // Validity check for api3 entry point
1778         if (newFlowConfig.isInternalFlow()) {
1779             String msg = "Invalid operation: Controller generated flow cannot be modified";
1780             String logMsg = msg + ": {}";
1781             log.warn(logMsg, newFlowConfig);
1782             return new Status(StatusCode.NOTACCEPTABLE, msg);
1783         }
1784
1785         // Validity Check
1786         Status status = newFlowConfig.validate(container);
1787         if (!status.isSuccess()) {
1788             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1789             newFlowConfig.setStatus(msg);
1790             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1791             return new Status(StatusCode.BADREQUEST, msg);
1792         }
1793
1794         FlowConfig oldFlowConfig = null;
1795         Integer index = null;
1796         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1797             FlowConfig entry = mapEntry.getValue();
1798             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1799                 oldFlowConfig = entry;
1800                 index = mapEntry.getKey();
1801                 break;
1802             }
1803         }
1804
1805         if (oldFlowConfig == null) {
1806             String msg = "Attempt to modify a non existing static flow";
1807             String logMsg = msg + ": {}";
1808             log.warn(logMsg, newFlowConfig);
1809             return new Status(StatusCode.NOTFOUND, msg);
1810         }
1811
1812         // Do not attempt to reinstall the flow, warn user
1813         if (newFlowConfig.equals(oldFlowConfig)) {
1814             String msg = "No modification detected";
1815             log.info("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1816             return new Status(StatusCode.SUCCESS, msg);
1817         }
1818
1819         // If flow is installed, program the network node
1820         status = new Status(StatusCode.SUCCESS, "Saved in config");
1821         if (oldFlowConfig.installInHw()) {
1822             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1823         }
1824
1825         // Update configuration database if programming was successful
1826         if (status.isSuccess()) {
1827             newFlowConfig.setStatus(status.getDescription());
1828             staticFlows.put(index, newFlowConfig);
1829         }
1830
1831         return status;
1832     }
1833
1834     @Override
1835     public Status toggleStaticFlowStatus(String name, Node node) {
1836         return toggleStaticFlowStatus(getStaticFlow(name, node));
1837     }
1838
1839     @Override
1840     public Status toggleStaticFlowStatus(FlowConfig config) {
1841         if (config == null) {
1842             String msg = "Invalid request: null flow config";
1843             log.warn(msg);
1844             return new Status(StatusCode.BADREQUEST, msg);
1845         }
1846         // Validity check for api3 entry point
1847         if (config.isInternalFlow()) {
1848             String msg = "Invalid operation: Controller generated flow cannot be modified";
1849             String logMsg = msg + ": {}";
1850             log.warn(logMsg, config);
1851             return new Status(StatusCode.NOTACCEPTABLE, msg);
1852         }
1853
1854         // Find the config entry
1855         Integer key = 0;
1856         FlowConfig target = null;
1857         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1858             FlowConfig conf = entry.getValue();
1859             if (conf.isByNameAndNodeIdEqual(config)) {
1860                 key = entry.getKey();
1861                 target = conf;
1862                 break;
1863             }
1864         }
1865         if (target != null) {
1866             Status status = target.validate(container);
1867             if (!status.isSuccess()) {
1868                 log.warn(status.getDescription());
1869                 return status;
1870             }
1871             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1872                                     .installFlowEntry(target.getFlowEntry());
1873             if (status.isSuccess()) {
1874                 // Update Configuration database
1875                 target.setStatus(StatusCode.SUCCESS.toString());
1876                 target.toggleInstallation();
1877                 staticFlows.put(key, target);
1878             }
1879             return status;
1880         }
1881
1882         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1883     }
1884
1885     /**
1886      * Reinsert all static flows entries in the cache to force cache updates in
1887      * the cluster. This is useful when only some parameters were changed in the
1888      * entries, like the status.
1889      *
1890      * @param node
1891      *            The node for which the static flow configurations have to be
1892      *            refreshed. If null, all nodes static flows will be refreshed.
1893      */
1894     private void refreshClusterStaticFlowsStatus(Node node) {
1895         // Refresh cluster cache
1896         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1897             if (node == null || entry.getValue().getNode().equals(node)) {
1898                 staticFlows.put(entry.getKey(), entry.getValue());
1899             }
1900         }
1901     }
1902
1903     /**
1904      * Uninstall all the non-internal Flow Entries present in the software view.
1905      * If requested, a copy of each original flow entry will be stored in the
1906      * inactive list so that it can be re-applied when needed (This is typically
1907      * the case when running in the default container and controller moved to
1908      * container mode) NOTE WELL: The routine as long as does a bulk change will
1909      * operate only on the entries for nodes locally attached so to avoid
1910      * redundant operations initiated by multiple nodes
1911      *
1912      * @param preserveFlowEntries
1913      *            if true, a copy of each original entry is stored in the
1914      *            inactive list
1915      */
1916     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1917         log.info("Uninstalling all non-internal flows");
1918
1919         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1920
1921         // Store entries / create target list
1922         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1923             FlowEntryInstall flowEntries = mapEntry.getValue();
1924             // Skip internal generated static flows
1925             if (!flowEntries.isInternal()) {
1926                 toRemove.add(flowEntries);
1927                 // Store the original entries if requested
1928                 if (preserveFlowEntries) {
1929                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1930                 }
1931             }
1932         }
1933
1934         // Now remove the entries
1935         for (FlowEntryInstall flowEntryHw : toRemove) {
1936             Node n = flowEntryHw.getNode();
1937             if (n != null && connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
1938                 Status status = this.removeEntryInternal(flowEntryHw, false);
1939                 if (!status.isSuccess()) {
1940                     log.trace("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1941                 }
1942             } else {
1943                 log.debug("Not removing entry {} because not connected locally, the remote guy will do it's job",
1944                         flowEntryHw);
1945             }
1946         }
1947     }
1948
1949     /**
1950      * Re-install all the Flow Entries present in the inactive list The inactive
1951      * list will be empty at the end of this call This function is called on the
1952      * default container instance of FRM only when the last container is deleted
1953      */
1954     private void reinstallAllFlowEntries() {
1955         log.info("Reinstalling all inactive flows");
1956
1957         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1958             this.addEntry(flowEntry, false);
1959         }
1960
1961         // Empty inactive list in any case
1962         inactiveFlows.clear();
1963     }
1964
1965     @Override
1966     public List<FlowConfig> getStaticFlows() {
1967         return getStaticFlowsOrderedList(staticFlows, staticFlowsOrdinal.get(0).intValue());
1968     }
1969
1970     // TODO: need to come out with a better algorithm for maintaining the order
1971     // of the configuration entries
1972     // with actual one, index associated to deleted entries cannot be reused and
1973     // map grows...
1974     private List<FlowConfig> getStaticFlowsOrderedList(ConcurrentMap<Integer, FlowConfig> flowMap, int maxKey) {
1975         List<FlowConfig> orderedList = new ArrayList<FlowConfig>();
1976         for (int i = 0; i <= maxKey; i++) {
1977             FlowConfig entry = flowMap.get(i);
1978             if (entry != null) {
1979                 orderedList.add(entry);
1980             }
1981         }
1982         return orderedList;
1983     }
1984
1985     @Override
1986     public FlowConfig getStaticFlow(String name, Node node) {
1987         ConcurrentMap.Entry<Integer, FlowConfig> entry = getStaticFlowEntry(name, node);
1988         if(entry != null) {
1989             return entry.getValue();
1990         }
1991         return null;
1992     }
1993
1994     @Override
1995     public List<FlowConfig> getStaticFlows(Node node) {
1996         List<FlowConfig> list = new ArrayList<FlowConfig>();
1997         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1998             if (entry.getValue().onNode(node)) {
1999                 list.add(entry.getValue());
2000             }
2001         }
2002         return list;
2003     }
2004
2005     @Override
2006     public List<String> getStaticFlowNamesForNode(Node node) {
2007         List<String> list = new ArrayList<String>();
2008         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2009             if (entry.getValue().onNode(node)) {
2010                 list.add(entry.getValue().getName());
2011             }
2012         }
2013         return list;
2014     }
2015
2016     @Override
2017     public List<Node> getListNodeWithConfiguredFlows() {
2018         Set<Node> set = new HashSet<Node>();
2019         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2020             set.add(entry.getValue().getNode());
2021         }
2022         return new ArrayList<Node>(set);
2023     }
2024
2025     @SuppressWarnings("unchecked")
2026     private void loadFlowConfiguration() {
2027         ObjectReader objReader = new ObjectReader();
2028         ConcurrentMap<Integer, FlowConfig> confList = (ConcurrentMap<Integer, FlowConfig>) objReader.read(this,
2029                 frmFileName);
2030
2031         ConcurrentMap<String, PortGroupConfig> pgConfig = (ConcurrentMap<String, PortGroupConfig>) objReader.read(this,
2032                 portGroupFileName);
2033
2034         if (pgConfig != null) {
2035             for (ConcurrentMap.Entry<String, PortGroupConfig> entry : pgConfig.entrySet()) {
2036                 addPortGroupConfig(entry.getKey(), entry.getValue().getMatchString(), true);
2037             }
2038         }
2039
2040         if (confList == null) {
2041             return;
2042         }
2043
2044         int maxKey = 0;
2045         for (Integer key : confList.keySet()) {
2046             if (key.intValue() > maxKey) {
2047                 maxKey = key.intValue();
2048             }
2049         }
2050
2051         for (FlowConfig conf : getStaticFlowsOrderedList(confList, maxKey)) {
2052             addStaticFlowInternal(conf, true);
2053         }
2054     }
2055
2056     @Override
2057     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2058         return ois.readObject();
2059     }
2060
2061     @Override
2062     public Status saveConfig() {
2063         return saveConfigInternal();
2064     }
2065
2066     private Status saveConfigInternal() {
2067         ObjectWriter objWriter = new ObjectWriter();
2068         ConcurrentMap<Integer, FlowConfig> nonDynamicFlows = new ConcurrentHashMap<Integer, FlowConfig>();
2069         for (Integer ordinal : staticFlows.keySet()) {
2070             FlowConfig config = staticFlows.get(ordinal);
2071             // Do not save dynamic and controller generated static flows
2072             if (config.isDynamic() || config.isInternalFlow()) {
2073                 continue;
2074             }
2075             nonDynamicFlows.put(ordinal, config);
2076         }
2077         objWriter.write(nonDynamicFlows, frmFileName);
2078         objWriter.write(new ConcurrentHashMap<String, PortGroupConfig>(portGroupConfigs), portGroupFileName);
2079         return new Status(StatusCode.SUCCESS, null);
2080     }
2081
2082     @Override
2083     public void subnetNotify(Subnet sub, boolean add) {
2084     }
2085
2086     /**
2087      * (non-Javadoc)
2088      *
2089      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2090      *      boolean)
2091      *
2092      *      This method can be called from within the OSGi framework context,
2093      *      given the programming operation can take sometime, it not good
2094      *      pratice to have in it's context operations that can take time,
2095      *      hence moving off to a different thread for async processing.
2096      */
2097     private ExecutorService executor;
2098     @Override
2099     public void modeChangeNotify(final Node node, final boolean proactive) {
2100         Callable<Status> modeChangeCallable = new Callable<Status>() {
2101             @Override
2102             public Status call() throws Exception {
2103                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2104
2105                 List<String> puntAction = new ArrayList<String>();
2106                 puntAction.add(ActionType.CONTROLLER.toString());
2107
2108                 FlowConfig allowARP = new FlowConfig();
2109                 allowARP.setInstallInHw(true);
2110                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2111                 allowARP.setPriority("1");
2112                 allowARP.setNode(node);
2113                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2114                         .toUpperCase());
2115                 allowARP.setActions(puntAction);
2116                 defaultConfigs.add(allowARP);
2117
2118                 FlowConfig allowLLDP = new FlowConfig();
2119                 allowLLDP.setInstallInHw(true);
2120                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2121                 allowLLDP.setPriority("1");
2122                 allowLLDP.setNode(node);
2123                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2124                         .toUpperCase());
2125                 allowLLDP.setActions(puntAction);
2126                 defaultConfigs.add(allowLLDP);
2127
2128                 List<String> dropAction = new ArrayList<String>();
2129                 dropAction.add(ActionType.DROP.toString());
2130
2131                 FlowConfig dropAllConfig = new FlowConfig();
2132                 dropAllConfig.setInstallInHw(true);
2133                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2134                         + FlowConfig.INTERNALSTATICFLOWEND);
2135                 dropAllConfig.setPriority("0");
2136                 dropAllConfig.setNode(node);
2137                 dropAllConfig.setActions(dropAction);
2138                 defaultConfigs.add(dropAllConfig);
2139
2140                 log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2141                 for (FlowConfig fc : defaultConfigs) {
2142                     Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2143                     if (status.isSuccess()) {
2144                         log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2145                     } else {
2146                         log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2147                                 fc.getName());
2148                     }
2149                 }
2150                 return new Status(StatusCode.SUCCESS);
2151             }
2152         };
2153
2154         /*
2155          * Execute the work outside the caller context, this could be an
2156          * expensive operation and we don't want to block the caller for it.
2157          */
2158         this.executor.submit(modeChangeCallable);
2159     }
2160
2161     /**
2162      * Remove from the databases all the flows installed on the node
2163      *
2164      * @param node
2165      */
2166     private void cleanDatabaseForNode(Node node) {
2167         log.info("Cleaning Flow database for Node {}", node);
2168         if (nodeFlows.containsKey(node)) {
2169             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2170
2171             for (FlowEntryInstall entry : toRemove) {
2172                 updateLocalDatabase(entry, false);
2173             }
2174         }
2175     }
2176
2177     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2178         if (nc == null) {
2179             return false;
2180         }
2181
2182         Match match = flow.getMatch();
2183         if (match.isPresent(MatchType.IN_PORT)) {
2184             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2185             if (matchPort.equals(nc)) {
2186                 return true;
2187             }
2188         }
2189         List<Action> actionsList = flow.getActions();
2190         if (actionsList != null) {
2191             for (Action action : actionsList) {
2192                 if (action instanceof Output) {
2193                     NodeConnector actionPort = ((Output) action).getPort();
2194                     if (actionPort.equals(nc)) {
2195                         return true;
2196                     }
2197                 }
2198             }
2199         }
2200         return false;
2201     }
2202
2203     @Override
2204     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2205         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2206     }
2207
2208     @Override
2209     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2210         boolean updateStaticFlowCluster = false;
2211
2212         switch (type) {
2213         case ADDED:
2214             break;
2215         case CHANGED:
2216             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2217             if (config != null) {
2218                 switch (config.getValue()) {
2219                 case Config.ADMIN_DOWN:
2220                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2221                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2222                     break;
2223                 case Config.ADMIN_UP:
2224                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2225                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2226                     break;
2227                 case Config.ADMIN_UNDEF:
2228                     break;
2229                 default:
2230                 }
2231             }
2232             break;
2233         case REMOVED:
2234             // This is the case where a switch port is removed from the SDN agent space
2235             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2236             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2237             break;
2238         default:
2239
2240         }
2241
2242         if (updateStaticFlowCluster) {
2243             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2244         }
2245     }
2246
2247     /*
2248      * It goes through the static flows configuration, it identifies the ones
2249      * which have the specified node connector as input or output port and
2250      * install them on the network node if they are marked to be installed in
2251      * hardware and their status shows they were not installed yet
2252      */
2253     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2254         boolean updated = false;
2255         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2256         for (FlowConfig flowConfig : flowConfigForNode) {
2257             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2258                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(StatusCode.SUCCESS.toString())) {
2259                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2260                     if (!status.isSuccess()) {
2261                         flowConfig.setStatus(status.getDescription());
2262                     } else {
2263                         flowConfig.setStatus(StatusCode.SUCCESS.toString());
2264                     }
2265                     updated = true;
2266                 }
2267             }
2268         }
2269         return updated;
2270     }
2271
2272     /*
2273      * Remove from the network node all the flows which have the specified node
2274      * connector as input or output port. If any of the flow entry is a static
2275      * flow, it updates the correspondent configuration.
2276      */
2277     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2278         boolean updated = false;
2279         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2280         if (nodeFlowEntries == null) {
2281             return updated;
2282         }
2283         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2284             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2285                 Status status = this.removeEntryInternal(fei, true);
2286                 if (!status.isSuccess()) {
2287                     continue;
2288                 }
2289                 /*
2290                  * If the flow entry is a static flow, then update its
2291                  * configuration
2292                  */
2293                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2294                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2295                     if (flowConfig != null) {
2296                         flowConfig.setStatus(PORTREMOVED);
2297                         updated = true;
2298                     }
2299                 }
2300             }
2301         }
2302         return updated;
2303     }
2304
2305     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2306         FlowConfig derivedFlow = new FlowConfig(original);
2307         derivedFlow.setDynamic(true);
2308         derivedFlow.setPortGroup(null);
2309         derivedFlow.setName(original.getName() + "_" + configName + "_" + port);
2310         derivedFlow.setIngressPort(port + "");
2311         return derivedFlow;
2312     }
2313
2314     private void addPortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2315         for (FlowConfig staticFlow : staticFlows.values()) {
2316             if (staticFlow.getPortGroup() == null) {
2317                 continue;
2318             }
2319             if ((staticFlow.getNode().equals(node)) && (staticFlow.getPortGroup().equals(config.getName()))) {
2320                 for (Short port : data.getPorts()) {
2321                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2322                     addStaticFlowInternal(derivedFlow, false);
2323                 }
2324             }
2325         }
2326     }
2327
2328     private void removePortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2329         for (FlowConfig staticFlow : staticFlows.values()) {
2330             if (staticFlow.getPortGroup() == null) {
2331                 continue;
2332             }
2333             if (staticFlow.getNode().equals(node) && staticFlow.getPortGroup().equals(config.getName())) {
2334                 for (Short port : data.getPorts()) {
2335                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2336                     removeStaticFlow(derivedFlow);
2337                 }
2338             }
2339         }
2340     }
2341
2342     @Override
2343     public void portGroupChanged(PortGroupConfig config, Map<Node, PortGroup> data, boolean add) {
2344         log.info("PortGroup Changed for: {} Data: {}", config, portGroupData);
2345         Map<Node, PortGroup> existingData = portGroupData.get(config);
2346         if (existingData != null) {
2347             for (Map.Entry<Node, PortGroup> entry : data.entrySet()) {
2348                 PortGroup existingPortGroup = existingData.get(entry.getKey());
2349                 if (existingPortGroup == null) {
2350                     if (add) {
2351                         existingData.put(entry.getKey(), entry.getValue());
2352                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2353                     }
2354                 } else {
2355                     if (add) {
2356                         existingPortGroup.getPorts().addAll(entry.getValue().getPorts());
2357                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2358                     } else {
2359                         existingPortGroup.getPorts().removeAll(entry.getValue().getPorts());
2360                         removePortGroupFlows(config, entry.getKey(), entry.getValue());
2361                     }
2362                 }
2363             }
2364         } else {
2365             if (add) {
2366                 portGroupData.put(config, data);
2367                 for (Node swid : data.keySet()) {
2368                     addPortGroupFlows(config, swid, data.get(swid));
2369                 }
2370             }
2371         }
2372     }
2373
2374     @Override
2375     public boolean addPortGroupConfig(String name, String regex, boolean restore) {
2376         PortGroupConfig config = portGroupConfigs.get(name);
2377         if (config != null) {
2378             return false;
2379         }
2380
2381         if ((portGroupProvider == null) && !restore) {
2382             return false;
2383         }
2384         if ((portGroupProvider != null) && (!portGroupProvider.isMatchCriteriaSupported(regex))) {
2385             return false;
2386         }
2387
2388         config = new PortGroupConfig(name, regex);
2389         portGroupConfigs.put(name, config);
2390         if (portGroupProvider != null) {
2391             portGroupProvider.createPortGroupConfig(config);
2392         }
2393         return true;
2394     }
2395
2396     @Override
2397     public boolean delPortGroupConfig(String name) {
2398         PortGroupConfig config = portGroupConfigs.get(name);
2399         if (config == null) {
2400             return false;
2401         }
2402
2403         if (portGroupProvider != null) {
2404             portGroupProvider.deletePortGroupConfig(config);
2405         }
2406         portGroupConfigs.remove(name);
2407         return true;
2408     }
2409
2410     @Override
2411     public Map<String, PortGroupConfig> getPortGroupConfigs() {
2412         return portGroupConfigs;
2413     }
2414
2415     public boolean isPortGroupSupported() {
2416         if (portGroupProvider == null) {
2417             return false;
2418         }
2419         return true;
2420     }
2421
2422     public void setIContainer(IContainer s) {
2423         this.container = s;
2424     }
2425
2426     public void unsetIContainer(IContainer s) {
2427         if (this.container == s) {
2428             this.container = null;
2429         }
2430     }
2431
2432     @Override
2433     public PortGroupProvider getPortGroupProvider() {
2434         return portGroupProvider;
2435     }
2436
2437     public void unsetPortGroupProvider(PortGroupProvider portGroupProvider) {
2438         this.portGroupProvider = null;
2439     }
2440
2441     public void setPortGroupProvider(PortGroupProvider portGroupProvider) {
2442         this.portGroupProvider = portGroupProvider;
2443         portGroupProvider.registerPortGroupChange(this);
2444         for (PortGroupConfig config : portGroupConfigs.values()) {
2445             portGroupProvider.createPortGroupConfig(config);
2446         }
2447     }
2448
2449     public void setFrmAware(IForwardingRulesManagerAware obj) {
2450         this.frmAware.add(obj);
2451     }
2452
2453     public void unsetFrmAware(IForwardingRulesManagerAware obj) {
2454         this.frmAware.remove(obj);
2455     }
2456
2457     void setClusterContainerService(IClusterContainerServices s) {
2458         log.debug("Cluster Service set");
2459         this.clusterContainerService = s;
2460     }
2461
2462     void unsetClusterContainerService(IClusterContainerServices s) {
2463         if (this.clusterContainerService == s) {
2464             log.debug("Cluster Service removed!");
2465             this.clusterContainerService = null;
2466         }
2467     }
2468
2469     private String getContainerName() {
2470         if (container == null) {
2471             return GlobalConstants.DEFAULT.toString();
2472         }
2473         return container.getName();
2474     }
2475
2476     /**
2477      * Function called by the dependency manager when all the required
2478      * dependencies are satisfied
2479      *
2480      */
2481     void init() {
2482         frmFileName = GlobalConstants.STARTUPHOME.toString() + "frm_staticflows_" + this.getContainerName() + ".conf";
2483         portGroupFileName = GlobalConstants.STARTUPHOME.toString() + "portgroup_" + this.getContainerName() + ".conf";
2484
2485         inContainerMode = false;
2486
2487         if (portGroupProvider != null) {
2488             portGroupProvider.registerPortGroupChange(this);
2489         }
2490
2491         cacheStartup();
2492
2493         registerWithOSGIConsole();
2494
2495         /*
2496          * If we are not the first cluster node to come up, do not initialize
2497          * the static flow entries ordinal
2498          */
2499         if (staticFlowsOrdinal.size() == 0) {
2500             staticFlowsOrdinal.put(0, Integer.valueOf(0));
2501         }
2502
2503         pendingEvents = new LinkedBlockingQueue<FRMEvent>();
2504
2505         // Initialize the event handler thread
2506         frmEventHandler = new Thread(new Runnable() {
2507             @Override
2508             public void run() {
2509                 while (!stopping) {
2510                     try {
2511                         final FRMEvent event = pendingEvents.take();
2512                         if (event == null) {
2513                             log.warn("Dequeued null event");
2514                             continue;
2515                         }
2516                         log.trace("Dequeued {} event", event.getClass().getSimpleName());
2517                         if (event instanceof NodeUpdateEvent) {
2518                             NodeUpdateEvent update = (NodeUpdateEvent) event;
2519                             Node node = update.getNode();
2520                             switch (update.getUpdateType()) {
2521                             case ADDED:
2522                                 addStaticFlowsToSwitch(node);
2523                                 break;
2524                             case REMOVED:
2525                                 cleanDatabaseForNode(node);
2526                                 updateStaticFlowConfigsOnNodeDown(node);
2527                                 break;
2528                             default:
2529                             }
2530                         } else if (event instanceof ErrorReportedEvent) {
2531                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
2532                             processErrorEvent(errEvent);
2533                         } else if (event instanceof WorkOrderEvent) {
2534                             /*
2535                              * Take care of handling the remote Work request
2536                              */
2537                             Runnable r = new Runnable() {
2538                                 @Override
2539                                 public void run() {
2540                                     WorkOrderEvent work = (WorkOrderEvent) event;
2541                                     FlowEntryDistributionOrder fe = work.getFe();
2542                                     if (fe != null) {
2543                                         logsync.trace("Executing the workOrder {}", fe);
2544                                         Status gotStatus = null;
2545                                         FlowEntryInstall feiCurrent = fe.getEntry();
2546                                         FlowEntryInstall feiNew = workOrder.get(fe);
2547                                         switch (fe.getUpType()) {
2548                                         case ADDED:
2549                                             gotStatus = addEntriesInternal(feiCurrent, false);
2550                                             break;
2551                                         case CHANGED:
2552                                             gotStatus = modifyEntryInternal(feiCurrent, feiNew, false);
2553                                             break;
2554                                         case REMOVED:
2555                                             gotStatus = removeEntryInternal(feiCurrent, false);
2556                                             break;
2557                                         }
2558                                         // Remove the Order
2559                                         workOrder.remove(fe);
2560                                         logsync.trace(
2561                                                 "The workOrder has been executed and now the status is being returned {}", fe);
2562                                         // Place the status
2563                                         workStatus.put(fe, gotStatus);
2564                                     } else {
2565                                         log.warn("Not expected null WorkOrder", work);
2566                                     }
2567                                 }
2568                             };
2569                             if(executor != null) {
2570                                 executor.execute(r);
2571                             }
2572                         } else if (event instanceof WorkStatusCleanup) {
2573                             /*
2574                              * Take care of handling the remote Work request
2575                              */
2576                             WorkStatusCleanup work = (WorkStatusCleanup) event;
2577                             FlowEntryDistributionOrder fe = work.getFe();
2578                             if (fe != null) {
2579                                 logsync.trace("The workStatus {} is being removed", fe);
2580                                 workStatus.remove(fe);
2581                             } else {
2582                                 log.warn("Not expected null WorkStatus", work);
2583                             }
2584                         }  else if (event instanceof ContainerFlowChangeEvent) {
2585                             /*
2586                              * Whether it is an addition or removal, we have to
2587                              * recompute the merged flows entries taking into
2588                              * account all the current container flows because
2589                              * flow merging is not an injective function
2590                              */
2591                             updateFlowsContainerFlow();
2592                         } else {
2593                             log.warn("Dequeued unknown event {}", event.getClass()
2594                                     .getSimpleName());
2595                         }
2596                     } catch (InterruptedException e) {
2597                         // clear pending events
2598                         pendingEvents.clear();
2599                     }
2600                 }
2601             }
2602         }, "FRM EventHandler Collector");
2603     }
2604
2605     /**
2606      * Function called by the dependency manager when at least one dependency
2607      * become unsatisfied or when the component is shutting down because for
2608      * example bundle is being stopped.
2609      *
2610      */
2611     void destroy() {
2612         // Interrupt the thread
2613         frmEventHandler.interrupt();
2614         // Clear the pendingEvents queue
2615         pendingEvents.clear();
2616         frmAware.clear();
2617         workMonitor.clear();
2618     }
2619
2620     /**
2621      * Function called by dependency manager after "init ()" is called and after
2622      * the services provided by the class are registered in the service registry
2623      *
2624      */
2625     void start() {
2626         /*
2627          * If running in default container, need to know if controller is in
2628          * container mode
2629          */
2630         if (GlobalConstants.DEFAULT.toString().equals(this.getContainerName())) {
2631             inContainerMode = containerManager.inContainerMode();
2632         }
2633
2634         // Initialize graceful stop flag
2635         stopping = false;
2636
2637         // Allocate the executor service
2638         this.executor = Executors.newFixedThreadPool(maxPoolSize);
2639
2640         // Start event handler thread
2641         frmEventHandler.start();
2642
2643         /*
2644          * Read startup and build database if we have not already gotten the
2645          * configurations synced from another node
2646          */
2647         if (staticFlows.isEmpty()) {
2648             loadFlowConfiguration();
2649         }
2650     }
2651
2652     /**
2653      * Function called by the dependency manager before the services exported by
2654      * the component are unregistered, this will be followed by a "destroy ()"
2655      * calls
2656      */
2657     void stop() {
2658         stopping = true;
2659         uninstallAllFlowEntries(false);
2660         // Shutdown executor
2661         this.executor.shutdownNow();
2662         // Now walk all the workMonitor and wake up the one sleeping because
2663         // destruction is happening
2664         for (FlowEntryDistributionOrder fe : workMonitor.keySet()) {
2665             FlowEntryDistributionOrderFutureTask task = workMonitor.get(fe);
2666             task.cancel(true);
2667         }
2668     }
2669
2670     public void setFlowProgrammerService(IFlowProgrammerService service) {
2671         this.programmer = service;
2672     }
2673
2674     public void unsetFlowProgrammerService(IFlowProgrammerService service) {
2675         if (this.programmer == service) {
2676             this.programmer = null;
2677         }
2678     }
2679
2680     public void setSwitchManager(ISwitchManager switchManager) {
2681         this.switchManager = switchManager;
2682     }
2683
2684     public void unsetSwitchManager(ISwitchManager switchManager) {
2685         if (this.switchManager == switchManager) {
2686             this.switchManager = null;
2687         }
2688     }
2689
2690     @Override
2691     public void tagUpdated(String containerName, Node n, short oldTag, short newTag, UpdateType t) {
2692         if (!container.getName().equals(containerName)) {
2693             return;
2694         }
2695     }
2696
2697     @Override
2698     public void containerFlowUpdated(String containerName, ContainerFlow previous, ContainerFlow current, UpdateType t) {
2699         if (!container.getName().equals(containerName)) {
2700             return;
2701         }
2702         log.trace("Container {}: Updating installed flows because of container flow change: {} {}",
2703                 container.getName(), t, current);
2704         ContainerFlowChangeEvent ev = new ContainerFlowChangeEvent(previous, current, t);
2705         pendingEvents.offer(ev);
2706     }
2707
2708     @Override
2709     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
2710         if (!container.getName().equals(containerName)) {
2711             return;
2712         }
2713
2714         boolean updateStaticFlowCluster = false;
2715
2716         switch (t) {
2717         case REMOVED:
2718             log.trace("Port {} was removed from container: uninstalling interested flows", nc);
2719             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nc);
2720             break;
2721         case ADDED:
2722             log.trace("Port {} was added to container: reinstall interested flows", nc);
2723             updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nc);
2724
2725             break;
2726         case CHANGED:
2727             break;
2728         default:
2729         }
2730
2731         if (updateStaticFlowCluster) {
2732             refreshClusterStaticFlowsStatus(nc.getNode());
2733         }
2734     }
2735
2736     @Override
2737     public void containerModeUpdated(UpdateType update) {
2738         // Only default container instance reacts on this event
2739         if (!container.getName().equals(GlobalConstants.DEFAULT.toString())) {
2740             return;
2741         }
2742         switch (update) {
2743         case ADDED:
2744             /*
2745              * Controller is moving to container mode. We are in the default
2746              * container context, we need to remove all our non-internal flows
2747              * to prevent any container isolation breakage. We also need to
2748              * preserve our flow so that they can be re-installed if we move
2749              * back to non container mode (no containers).
2750              */
2751             this.inContainerMode = true;
2752             this.uninstallAllFlowEntries(true);
2753             break;
2754         case REMOVED:
2755             this.inContainerMode = false;
2756             this.reinstallAllFlowEntries();
2757             break;
2758         default:
2759         }
2760
2761         // Update our configuration DB
2762         updateStaticFlowConfigsOnContainerModeChange(update);
2763     }
2764
2765     protected abstract class FRMEvent {
2766
2767     }
2768
2769     private class NodeUpdateEvent extends FRMEvent {
2770         private final Node node;
2771         private final UpdateType update;
2772
2773         public NodeUpdateEvent(UpdateType update, Node node) {
2774             this.update = update;
2775             this.node = node;
2776         }
2777
2778         public UpdateType getUpdateType() {
2779             return update;
2780         }
2781
2782         public Node getNode() {
2783             return node;
2784         }
2785     }
2786
2787     private class ErrorReportedEvent extends FRMEvent {
2788         private final long rid;
2789         private final Node node;
2790         private final Object error;
2791
2792         public ErrorReportedEvent(long rid, Node node, Object error) {
2793             this.rid = rid;
2794             this.node = node;
2795             this.error = error;
2796         }
2797
2798         public long getRequestId() {
2799             return rid;
2800         }
2801
2802         public Object getError() {
2803             return error;
2804         }
2805
2806         public Node getNode() {
2807             return node;
2808         }
2809     }
2810
2811     private class WorkOrderEvent extends FRMEvent {
2812         private FlowEntryDistributionOrder fe;
2813         private FlowEntryInstall newEntry;
2814
2815         /**
2816          * @param fe
2817          * @param newEntry
2818          */
2819         WorkOrderEvent(FlowEntryDistributionOrder fe, FlowEntryInstall newEntry) {
2820             this.fe = fe;
2821             this.newEntry = newEntry;
2822         }
2823
2824         /**
2825          * @return the fe
2826          */
2827         public FlowEntryDistributionOrder getFe() {
2828             return fe;
2829         }
2830
2831         /**
2832          * @return the newEntry
2833          */
2834         public FlowEntryInstall getNewEntry() {
2835             return newEntry;
2836         }
2837     }
2838     private class ContainerFlowChangeEvent extends FRMEvent {
2839         private final ContainerFlow previous;
2840         private final ContainerFlow current;
2841         private final UpdateType type;
2842
2843         public ContainerFlowChangeEvent(ContainerFlow previous, ContainerFlow current, UpdateType type) {
2844             this.previous = previous;
2845             this.current = current;
2846             this.type = type;
2847         }
2848
2849         public ContainerFlow getPrevious() {
2850             return this.previous;
2851         }
2852
2853         public ContainerFlow getCurrent() {
2854             return this.current;
2855         }
2856
2857         public UpdateType getType() {
2858             return this.type;
2859         }
2860     }
2861
2862
2863     private class WorkStatusCleanup extends FRMEvent {
2864         private FlowEntryDistributionOrder fe;
2865
2866         /**
2867          * @param fe
2868          */
2869         WorkStatusCleanup(FlowEntryDistributionOrder fe) {
2870             this.fe = fe;
2871         }
2872
2873         /**
2874          * @return the fe
2875          */
2876         public FlowEntryDistributionOrder getFe() {
2877             return fe;
2878         }
2879     }
2880
2881     /*
2882      * OSGI COMMANDS
2883      */
2884     @Override
2885     public String getHelp() {
2886         StringBuffer help = new StringBuffer();
2887         return help.toString();
2888     }
2889
2890     @Override
2891     public Status saveConfiguration() {
2892         return saveConfig();
2893     }
2894
2895     public void _frmNodeFlows(CommandInterpreter ci) {
2896         String nodeId = ci.nextArgument();
2897         Node node = Node.fromString(nodeId);
2898         if (node == null) {
2899             ci.println("frmNodeFlows <node> [verbose]");
2900             return;
2901         }
2902         boolean verbose = false;
2903         String verboseCheck = ci.nextArgument();
2904         if (verboseCheck != null) {
2905             verbose = verboseCheck.equals("true");
2906         }
2907
2908         if (!nodeFlows.containsKey(node)) {
2909             return;
2910         }
2911         // Dump per node database
2912         for (FlowEntryInstall entry : nodeFlows.get(node)) {
2913             if (!verbose) {
2914                 ci.println(node + " " + installedSwView.get(entry).getFlowName());
2915             } else {
2916                 ci.println(node + " " + installedSwView.get(entry).toString());
2917             }
2918         }
2919     }
2920
2921     public void _frmGroupFlows(CommandInterpreter ci) {
2922         String group = ci.nextArgument();
2923         if (group == null) {
2924             ci.println("frmGroupFlows <group> [verbose]");
2925             return;
2926         }
2927         boolean verbose = false;
2928         String verboseCheck = ci.nextArgument();
2929         if (verboseCheck != null) {
2930             verbose = verboseCheck.equalsIgnoreCase("true");
2931         }
2932
2933         if (!groupFlows.containsKey(group)) {
2934             return;
2935         }
2936         // Dump per node database
2937         ci.println("Group " + group + ":\n");
2938         for (FlowEntryInstall flowEntry : groupFlows.get(group)) {
2939             if (!verbose) {
2940                 ci.println(flowEntry.getNode() + " " + flowEntry.getFlowName());
2941             } else {
2942                 ci.println(flowEntry.getNode() + " " + flowEntry.toString());
2943             }
2944         }
2945     }
2946
2947     public void _frmProcessErrorEvent(CommandInterpreter ci) throws UnknownHostException {
2948         Node node = null;
2949         long reqId = 0L;
2950         String nodeId = ci.nextArgument();
2951         if (nodeId == null) {
2952             ci.print("Node id not specified");
2953             return;
2954         }
2955         String requestId = ci.nextArgument();
2956         if (requestId == null) {
2957             ci.print("Request id not specified");
2958             return;
2959         }
2960         try {
2961             node = NodeCreator.createOFNode(Long.valueOf(nodeId));
2962         } catch (NumberFormatException e) {
2963             ci.print("Node id not a number");
2964             return;
2965         }
2966         try {
2967             reqId = Long.parseLong(requestId);
2968         } catch (NumberFormatException e) {
2969             ci.print("Request id not a number");
2970             return;
2971         }
2972         // null for error object is good enough for now
2973         ErrorReportedEvent event = new ErrorReportedEvent(reqId, node, null);
2974         this.processErrorEvent(event);
2975     }
2976
2977     @Override
2978     public void flowRemoved(Node node, Flow flow) {
2979         log.trace("Received flow removed notification on {} for {}", node, flow);
2980
2981         // For flow entry identification, only node, match and priority matter
2982         FlowEntryInstall test = new FlowEntryInstall(new FlowEntry("", "", flow, node), null);
2983         FlowEntryInstall installedEntry = this.installedSwView.get(test);
2984         if (installedEntry == null) {
2985             log.trace("Entry is not known to us");
2986             return;
2987         }
2988
2989         // Update Static flow status
2990         Integer key = 0;
2991         FlowConfig target = null;
2992         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
2993             FlowConfig conf = entry.getValue();
2994             if (conf.isByNameAndNodeIdEqual(installedEntry.getFlowName(), node)) {
2995                 key = entry.getKey();
2996                 target = conf;
2997                 break;
2998             }
2999         }
3000         if (target != null) {
3001             // Update Configuration database
3002             target.toggleInstallation();
3003             target.setStatus(StatusCode.SUCCESS.toString());
3004             staticFlows.put(key, target);
3005         }
3006
3007         // Update software views
3008         this.updateLocalDatabase(installedEntry, false);
3009     }
3010
3011     @Override
3012     public void flowErrorReported(Node node, long rid, Object err) {
3013         log.trace("Got error {} for message rid {} from node {}", new Object[] { err, rid, node });
3014         pendingEvents.offer(new ErrorReportedEvent(rid, node, err));
3015     }
3016
3017     private void processErrorEvent(ErrorReportedEvent event) {
3018         Node node = event.getNode();
3019         long rid = event.getRequestId();
3020         Object error = event.getError();
3021         String errorString = (error == null) ? "Not provided" : error.toString();
3022         /*
3023          * If this was for a flow install, remove the corresponding entry from
3024          * the software view. If it was a Looking for the rid going through the
3025          * software database. TODO: A more efficient rid <-> FlowEntryInstall
3026          * mapping will have to be added in future
3027          */
3028         FlowEntryInstall target = null;
3029         List<FlowEntryInstall> flowEntryInstallList = nodeFlows.get(node);
3030         // flowEntryInstallList could be null.
3031         // so check for it.
3032         if(flowEntryInstallList != null) {
3033             for (FlowEntryInstall index : flowEntryInstallList) {
3034                 FlowEntryInstall entry = installedSwView.get(index);
3035                 if(entry != null) {
3036                     if (entry.getRequestId() == rid) {
3037                         target = entry;
3038                         break;
3039                     }
3040                 }
3041             }
3042         }
3043         if (target != null) {
3044             // This was a flow install, update database
3045             this.updateLocalDatabase(target, false);
3046             // also update the config
3047             if(FlowConfig.STATICFLOWGROUP.equals(target.getGroupName())) {
3048                 ConcurrentMap.Entry<Integer, FlowConfig> staticFlowEntry = getStaticFlowEntry(target.getFlowName(),target.getNode());
3049                 // staticFlowEntry should never be null.
3050                 // the null check is just an extra defensive check.
3051                 if(staticFlowEntry != null) {
3052                     staticFlows.remove(staticFlowEntry.getKey());
3053                 }
3054             }
3055         }
3056
3057         // Notify listeners
3058         if (frmAware != null) {
3059             synchronized (frmAware) {
3060                 for (IForwardingRulesManagerAware frma : frmAware) {
3061                     try {
3062                         frma.requestFailed(rid, errorString);
3063                     } catch (Exception e) {
3064                         log.warn("Failed to notify {}", frma);
3065                     }
3066                 }
3067             }
3068         }
3069     }
3070
3071     @Override
3072     public Status solicitStatusResponse(Node node, boolean blocking) {
3073         Status rv = new Status(StatusCode.INTERNALERROR);
3074
3075         if (this.programmer != null) {
3076             if (blocking) {
3077                 rv = programmer.syncSendBarrierMessage(node);
3078             } else {
3079                 rv = programmer.asyncSendBarrierMessage(node);
3080             }
3081         }
3082
3083         return rv;
3084     }
3085
3086     public void unsetIConnectionManager(IConnectionManager s) {
3087         if (s == this.connectionManager) {
3088             this.connectionManager = null;
3089         }
3090     }
3091
3092     public void setIConnectionManager(IConnectionManager s) {
3093         this.connectionManager = s;
3094     }
3095
3096     public void unsetIContainerManager(IContainerManager s) {
3097         if (s == this.containerManager) {
3098             this.containerManager = null;
3099         }
3100     }
3101
3102     public void setIContainerManager(IContainerManager s) {
3103         this.containerManager = s;
3104     }
3105
3106     @Override
3107     public void entryCreated(Object key, String cacheName, boolean originLocal) {
3108         /*
3109          * Do nothing
3110          */
3111     }
3112
3113     @Override
3114     public void entryUpdated(Object key, Object new_value, String cacheName, boolean originLocal) {
3115         if (originLocal) {
3116             /*
3117              * Local updates are of no interest
3118              */
3119             return;
3120         }
3121         if (cacheName.equals(WORK_ORDER_CACHE)) {
3122             logsync.trace("Got a WorkOrderCacheUpdate for {}", key);
3123             /*
3124              * This is the case of one workOrder becoming available, so we need
3125              * to dispatch the work to the appropriate handler
3126              */
3127             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3128             FlowEntryInstall fei = fe.getEntry();
3129             if (fei == null) {
3130                 return;
3131             }
3132             Node n = fei.getNode();
3133             if (connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
3134                 logsync.trace("workOrder for fe {} processed locally", fe);
3135                 // I'm the controller in charge for the request, queue it for
3136                 // processing
3137                 pendingEvents.offer(new WorkOrderEvent(fe, (FlowEntryInstall) new_value));
3138             }
3139         } else if (cacheName.equals(WORK_STATUS_CACHE)) {
3140             logsync.trace("Got a WorkStatusCacheUpdate for {}", key);
3141             /*
3142              * This is the case of one workOrder being completed and a status
3143              * returned
3144              */
3145             FlowEntryDistributionOrder fe = (FlowEntryDistributionOrder) key;
3146             /*
3147              * Check if the order was initiated by this controller in that case
3148              * we need to actually look at the status returned
3149              */
3150             if (fe.getRequestorController()
3151                     .equals(clusterContainerService.getMyAddress())) {
3152                 FlowEntryDistributionOrderFutureTask fet = workMonitor.remove(fe);
3153                 if (fet != null) {
3154                     logsync.trace("workStatus response is for us {}", fe);
3155                     // Signal we got the status
3156                     fet.gotStatus(fe, workStatus.get(fe));
3157                     pendingEvents.offer(new WorkStatusCleanup(fe));
3158                 }
3159             }
3160         }
3161     }
3162
3163     @Override
3164     public void entryDeleted(Object key, String cacheName, boolean originLocal) {
3165         /*
3166          * Do nothing
3167          */
3168     }
3169
3170     /**
3171      * {@inheritDoc}
3172      */
3173     @Override
3174     public List<FlowEntry> getFlowEntriesForNode(Node node) {
3175         List<FlowEntry> list = new ArrayList<FlowEntry>();
3176         if (node != null) {
3177             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
3178                 if (node.equals(entry.getKey().getNode())) {
3179                     list.add(entry.getKey().clone());
3180                 }
3181             }
3182         }
3183         return list;
3184     }
3185
3186     /**
3187      * {@inheritDoc}
3188      */
3189     @Override
3190     public List<FlowEntry> getInstalledFlowEntriesForNode(Node node) {
3191         List<FlowEntry> list = new ArrayList<FlowEntry>();
3192         if (node != null) {
3193             List<FlowEntryInstall> flowEntryInstallList = this.nodeFlows.get(node);
3194             if(flowEntryInstallList != null) {
3195                 for(FlowEntryInstall fi: flowEntryInstallList) {
3196                     list.add(fi.getInstall().clone());
3197                 }
3198             }
3199         }
3200         return list;
3201     }
3202 }