665ba7c635781968458dce7b026fc4bf6477d449
[controller.git] / opendaylight / forwardingrulesmanager / implementation / src / main / java / org / opendaylight / controller / forwardingrulesmanager / internal / ForwardingRulesManager.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.forwardingrulesmanager.internal;
10
11 import java.io.FileNotFoundException;
12 import java.io.IOException;
13 import java.io.ObjectInputStream;
14 import java.net.UnknownHostException;
15 import java.util.ArrayList;
16 import java.util.Collections;
17 import java.util.EnumSet;
18 import java.util.HashSet;
19 import java.util.List;
20 import java.util.Map;
21 import java.util.Map.Entry;
22 import java.util.Set;
23 import java.util.concurrent.BlockingQueue;
24 import java.util.concurrent.Callable;
25 import java.util.concurrent.ConcurrentHashMap;
26 import java.util.concurrent.ConcurrentMap;
27 import java.util.concurrent.ExecutionException;
28 import java.util.concurrent.ExecutorService;
29 import java.util.concurrent.Executors;
30 import java.util.concurrent.LinkedBlockingQueue;
31
32 import org.eclipse.osgi.framework.console.CommandInterpreter;
33 import org.eclipse.osgi.framework.console.CommandProvider;
34 import org.opendaylight.controller.clustering.services.CacheConfigException;
35 import org.opendaylight.controller.clustering.services.CacheExistException;
36 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
37 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
38 import org.opendaylight.controller.clustering.services.IClusterServices;
39 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
40 import org.opendaylight.controller.connectionmanager.IConnectionManager;
41 import org.opendaylight.controller.containermanager.IContainerManager;
42 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
43 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
44 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
45 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
46 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
47 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
48 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
49 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
50 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
51 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
52 import org.opendaylight.controller.sal.action.Action;
53 import org.opendaylight.controller.sal.action.ActionType;
54 import org.opendaylight.controller.sal.action.Output;
55 import org.opendaylight.controller.sal.connection.ConnectionLocality;
56 import org.opendaylight.controller.sal.core.Config;
57 import org.opendaylight.controller.sal.core.ContainerFlow;
58 import org.opendaylight.controller.sal.core.IContainer;
59 import org.opendaylight.controller.sal.core.IContainerLocalListener;
60 import org.opendaylight.controller.sal.core.Node;
61 import org.opendaylight.controller.sal.core.NodeConnector;
62 import org.opendaylight.controller.sal.core.Property;
63 import org.opendaylight.controller.sal.core.UpdateType;
64 import org.opendaylight.controller.sal.flowprogrammer.Flow;
65 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
66 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
67 import org.opendaylight.controller.sal.match.Match;
68 import org.opendaylight.controller.sal.match.MatchType;
69 import org.opendaylight.controller.sal.utils.EtherTypes;
70 import org.opendaylight.controller.sal.utils.GlobalConstants;
71 import org.opendaylight.controller.sal.utils.IObjectReader;
72 import org.opendaylight.controller.sal.utils.NodeCreator;
73 import org.opendaylight.controller.sal.utils.ObjectReader;
74 import org.opendaylight.controller.sal.utils.ObjectWriter;
75 import org.opendaylight.controller.sal.utils.Status;
76 import org.opendaylight.controller.sal.utils.StatusCode;
77 import org.opendaylight.controller.switchmanager.IInventoryListener;
78 import org.opendaylight.controller.switchmanager.ISwitchManager;
79 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
80 import org.opendaylight.controller.switchmanager.Subnet;
81 import org.osgi.framework.BundleContext;
82 import org.osgi.framework.FrameworkUtil;
83 import org.slf4j.Logger;
84 import org.slf4j.LoggerFactory;
85
86 /**
87  * Class that manages forwarding rule installation and removal per container of
88  * the network. It also maintains the central repository of all the forwarding
89  * rules installed on the network nodes.
90  */
91 public class ForwardingRulesManager implements
92         IForwardingRulesManager,
93         PortGroupChangeListener,
94         IContainerLocalListener,
95         ISwitchManagerAware,
96         IConfigurationContainerAware,
97         IInventoryListener,
98         IObjectReader,
99         ICacheUpdateAware<Object,Object>,
100         CommandProvider,
101         IFlowProgrammerListener {
102
103     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
104     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
105     private static final String PORT_REMOVED = "Port removed";
106     private static final String NODE_DOWN = "Node is Down";
107     private static final String INVALID_FLOW_ENTRY = "Invalid FlowEntry";
108     private String frmFileName;
109     private String portGroupFileName;
110     private ConcurrentMap<Integer, FlowConfig> staticFlows;
111     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
112     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
113     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
114     private ConcurrentMap<String, Object> TSPolicies;
115     private IContainerManager containerManager;
116     private boolean inContainerMode; // being used by global instance only
117     protected boolean stopping;
118
119     /*
120      * Flow database. It's the software view of what was requested to install
121      * and what is installed on the switch. It is indexed by the entry itself.
122      * The entry's hashcode resumes the network node index, the flow's priority
123      * and the flow's match. The value element is a class which contains the
124      * flow entry pushed by the applications modules and the respective
125      * container flow merged version. In absence of container flows, the two
126      * flow entries are the same.
127      */
128     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
129     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
130     /*
131      * Per node and per group indexing
132      */
133     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
134     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
135
136     /*
137      * Inactive flow list. This is for the global instance of FRM It will
138      * contain all the flow entries which were installed on the global container
139      * when the first container is created.
140      */
141     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
142
143     private IContainer container;
144     private Set<IForwardingRulesManagerAware> frmAware =
145         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
146     private PortGroupProvider portGroupProvider;
147     private IFlowProgrammerService programmer;
148     private IClusterContainerServices clusterContainerService = null;
149     private ISwitchManager switchManager;
150     private Thread frmEventHandler;
151     protected BlockingQueue<FRMEvent> pendingEvents;
152
153     // Distributes FRM programming in the cluster
154     private IConnectionManager connectionManager;
155
156     /*
157      * Name clustered caches used to support FRM entry distribution these are by
158      * necessity non-transactional as long as need to be able to synchronize
159      * states also while a transaction is in progress
160      */
161     static final String WORK_ORDER_CACHE = "frm.workOrder";
162     static final String WORK_STATUS_CACHE = "frm.workStatus";
163     static final String ORIGINAL_SW_VIEW_CACHE = "frm.originalSwView";
164     static final String INSTALLED_SW_VIEW_CACHE = "frm.installedSwView";
165
166     /*
167      * Data structure responsible for distributing the FlowEntryInstall requests
168      * in the cluster. The key value is entry that is being either Installed or
169      * Updated or Delete. The value field is the same of the key value in case
170      * of Installation or Deletion, it's the new entry in case of Modification,
171      * this because the clustering caches don't allow null values.
172      *
173      * The logic behind this data structure is that the controller that initiate
174      * the request will place the order here, someone will pick it and then will
175      * remove from this data structure because is being served.
176      *
177      * TODO: We need to have a way to cleanup this data structure if entries are
178      * not picked by anyone, which is always a case can happen especially on
179      * Node disconnect cases.
180      */
181     protected ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
182
183     /*
184      * Data structure responsible for retrieving the results of the workOrder
185      * submitted to the cluster.
186      *
187      * The logic behind this data structure is that the controller that has
188      * executed the order will then place the result in workStatus signaling
189      * that there was a success or a failure.
190      *
191      * TODO: The workStatus entries need to have a lifetime associated in case
192      * of requestor controller leaving the cluster.
193      */
194     protected ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
195
196     /*
197      * Local Map used to hold the Future which a caller can use to monitor for
198      * completion
199      */
200     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
201             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
202
203     /*
204      * Max pool size for the executor
205      */
206     private static final int maxPoolSize = 10;
207
208     /**
209      * @param e
210      *            Entry being installed/updated/removed
211      * @param u
212      *            New entry will be placed after the update operation. Valid
213      *            only for UpdateType.CHANGED, null for all the other cases
214      * @param t
215      *            Type of update
216      * @return a Future object for monitoring the progress of the result, or
217      *         null in case the processing should take place locally
218      */
219     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
220             UpdateType t) {
221         // A null entry it's an unexpected condition, anyway it's safe to keep
222         // the handling local
223         if (e == null) {
224             return null;
225         }
226
227         Node n = e.getNode();
228         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
229             // Create the work order and distribute it
230             FlowEntryDistributionOrder fe =
231                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
232             // First create the monitor job
233             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
234             logsync.trace("Node {} not local so sending fe {}", n, fe);
235             workMonitor.put(fe, ret);
236             if (t.equals(UpdateType.CHANGED)) {
237                 // Then distribute the work
238                 workOrder.put(fe, u);
239             } else {
240                 // Then distribute the work
241                 workOrder.put(fe, e);
242             }
243             logsync.trace("WorkOrder requested");
244             // Now create an Handle to monitor the execution of the operation
245             return ret;
246         }
247
248         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
249         return null;
250     }
251
252     /**
253      * Adds a flow entry onto the network node It runs various validity checks
254      * and derive the final container flows merged entries that will be
255      * attempted to be installed
256      *
257      * @param flowEntry
258      *            the original flow entry application requested to add
259      * @param async
260      *            the flag indicating if this is a asynchronous request
261      * @return the status of this request. In case of asynchronous call, it will
262      *         contain the unique id assigned to this request
263      */
264     private Status addEntry(FlowEntry flowEntry, boolean async) {
265
266         // Sanity Check
267         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
268             String logMsg = INVALID_FLOW_ENTRY + ": {}";
269             log.warn(logMsg, flowEntry);
270             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
271         }
272
273         /*
274          * Redundant Check: Check if the request is a redundant one from the
275          * same application the flowEntry is equal to an existing one. Given we
276          * do not have an application signature in the requested FlowEntry yet,
277          * we are here detecting the above condition by comparing the flow
278          * names, if set. If they are equal to the installed flow, most likely
279          * this is a redundant installation request from the same application
280          * and we can silently return success
281          *
282          * TODO: in future a sort of application reference list mechanism will
283          * be added to the FlowEntry so that exact flow can be used by different
284          * applications.
285          */
286         FlowEntry present = this.originalSwView.get(flowEntry);
287         if (present != null) {
288             boolean sameFlow = present.getFlow().equals(flowEntry.getFlow());
289             boolean sameApp = present.getFlowName() != null && present.getFlowName().equals(flowEntry.getFlowName());
290             if (sameFlow && sameApp) {
291                 log.trace("Skipping redundant request for flow {} on node {}", flowEntry.getFlowName(),
292                         flowEntry.getNode());
293                 return new Status(StatusCode.SUCCESS, "Entry is already present");
294             }
295         }
296
297         /*
298          * Derive the container flow merged entries to install In presence of N
299          * container flows, we may end up with N different entries to install...
300          */
301         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
302
303         // Container Flow conflict Check
304         if (toInstallList.isEmpty()) {
305             String msg = "Flow Entry conflicts with all Container Flows";
306             String logMsg = msg + ": {}";
307             log.warn(logMsg, flowEntry);
308             return new Status(StatusCode.CONFLICT, msg);
309         }
310
311         // Derive the list of entries good to be installed
312         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
313         for (FlowEntryInstall entry : toInstallList) {
314             // Conflict Check: Verify new entry would not overwrite existing
315             // ones
316             if (this.installedSwView.containsKey(entry)) {
317                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
318                 log.trace("Aborting to install {}", entry);
319                 continue;
320             }
321             toInstallSafe.add(entry);
322         }
323
324         // Declare failure if all the container flow merged entries clash with
325         // existing entries
326         if (toInstallSafe.size() == 0) {
327             String msg = "A flow with same match and priority exists on the target node";
328             String logMsg = msg + ": {}";
329             log.warn(logMsg, flowEntry);
330             return new Status(StatusCode.CONFLICT, msg);
331         }
332
333         // Try to install an entry at the time
334         Status error = new Status(null, null);
335         Status succeded = null;
336         boolean oneSucceded = false;
337         for (FlowEntryInstall installEntry : toInstallSafe) {
338
339             // Install and update database
340             Status ret = addEntriesInternal(installEntry, async);
341
342             if (ret.isSuccess()) {
343                 oneSucceded = true;
344                 /*
345                  * The first successful status response will be returned For the
346                  * asynchronous call, we can discard the container flow
347                  * complication for now and assume we will always deal with one
348                  * flow only per request
349                  */
350                 succeded = ret;
351             } else {
352                 error = ret;
353                 log.trace("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
354             }
355         }
356
357         return (oneSucceded) ? succeded : error;
358     }
359
360     /**
361      * Given a flow entry and the list of container flows, it returns the list
362      * of container flow merged flow entries good to be installed on this
363      * container. If the list of container flows is null or empty, the install
364      * entry list will contain only one entry, the original flow entry. If the
365      * flow entry is congruent with all the N container flows, then the output
366      * install entry list will contain N entries. If the output list is empty,
367      * it means the passed flow entry conflicts with all the container flows.
368      *
369      * @param cFlowList
370      *            The list of container flows
371      * @return the list of container flow merged entries good to be installed on
372      *         this container
373      */
374     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
375         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
376
377         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
378             // No container flows => entry good to be installed unchanged
379             toInstallList.add(new FlowEntryInstall(request.clone(), null));
380         } else {
381             // Create the list of entries to be installed. If the flow entry is
382             // not congruent with any container flow, no install entries will be
383             // created
384             for (ContainerFlow cFlow : container.getContainerFlows()) {
385                 if (cFlow.allowsFlow(request.getFlow())) {
386                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
387                 }
388             }
389         }
390         return toInstallList;
391     }
392
393     /**
394      * Modify a flow entry with a new one It runs various validity check and
395      * derive the final container flows merged flow entries to work with
396      *
397      * @param currentFlowEntry
398      * @param newFlowEntry
399      * @param async
400      *            the flag indicating if this is a asynchronous request
401      * @return the status of this request. In case of asynchronous call, it will
402      *         contain the unique id assigned to this request
403      */
404     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
405         Status retExt;
406
407         // Sanity checks
408         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
409                 || newFlowEntry.getNode() == null || newFlowEntry.getFlow() == null) {
410             String msg = "Modify: " + INVALID_FLOW_ENTRY;
411             String logMsg = msg + ": {} or {}";
412             log.warn(logMsg, currentFlowEntry, newFlowEntry);
413             return new Status(StatusCode.NOTACCEPTABLE, msg);
414         }
415         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
416                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
417             String msg = "Modify: Incompatible Flow Entries";
418             String logMsg = msg + ": {} and {}";
419             log.warn(logMsg, currentFlowEntry, newFlowEntry);
420             return new Status(StatusCode.NOTACCEPTABLE, msg);
421         }
422
423         // Equality Check
424         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
425             String msg = "Modify skipped as flows are the same";
426             String logMsg = msg + ": {} and {}";
427             log.debug(logMsg, currentFlowEntry, newFlowEntry);
428             return new Status(StatusCode.SUCCESS, msg);
429         }
430
431         /*
432          * Conflict Check: Verify the new entry would not conflict with an
433          * existing one. This is a loose check on the previous original flow
434          * entry requests. No check on the container flow merged flow entries
435          * (if any) yet
436          */
437         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
438         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
439             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
440             String logMsg = msg + ": {}";
441             log.warn(logMsg, currentFlowEntry);
442             return new Status(StatusCode.CONFLICT, msg);
443         }
444
445         // Derive the installed and toInstall entries
446         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
447                 container.getContainerFlows());
448         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
449
450         if (toInstallList.isEmpty()) {
451             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
452             String logMsg = msg + ": {}";
453             log.warn(logMsg, newFlowEntry);
454             log.warn(msg);
455             return new Status(StatusCode.CONFLICT, msg);
456         }
457
458         /*
459          * If the two list sizes differ, it means the new flow entry does not
460          * satisfy the same number of container flows the current entry does.
461          * This is only possible when the new entry and current entry have
462          * different match. In this scenario the modification would ultimately
463          * be handled as a remove and add operations in the protocol plugin.
464          *
465          * Also, if any of the new flow entries would clash with an existing
466          * one, we cannot proceed with the modify operation, because it would
467          * fail for some entries and leave stale entries on the network node.
468          * Modify path can be taken only if it can be performed completely, for
469          * all entries.
470          *
471          * So, for the above two cases, to simplify, let's decouple the modify
472          * in: 1) remove current entries 2) install new entries
473          */
474         Status succeeded = null;
475         boolean decouple = false;
476         if (installedList.size() != toInstallList.size()) {
477             log.info("Modify: New flow entry does not satisfy the same "
478                     + "number of container flows as the original entry does");
479             decouple = true;
480         }
481         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
482         for (FlowEntryInstall installEntry : toInstallList) {
483             /*
484              * Conflict Check: Verify the new entry would not overwrite another
485              * existing one
486              */
487             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
488             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
489                 log.info("Modify: new container flow merged flow entry clashes with existing flow");
490                 decouple = true;
491             } else {
492                 toInstallSafe.add(installEntry);
493             }
494         }
495
496         if (decouple) {
497             // Remove current entries
498             for (FlowEntryInstall currEntry : installedList) {
499                 this.removeEntryInternal(currEntry, async);
500             }
501             // Install new entries
502             for (FlowEntryInstall newEntry : toInstallSafe) {
503                 succeeded = this.addEntriesInternal(newEntry, async);
504             }
505         } else {
506             /*
507              * The two list have the same size and the entries to install do not
508              * clash with any existing flow on the network node. We assume here
509              * (and might be wrong) that the same container flows that were
510              * satisfied by the current entries are the same that are satisfied
511              * by the new entries. Let's take the risk for now.
512              *
513              * Note: modification has to be complete. If any entry modification
514              * fails, we need to stop, restore the already modified entries, and
515              * declare failure.
516              */
517             Status retModify = null;
518             int i = 0;
519             int size = toInstallList.size();
520             while (i < size) {
521                 // Modify and update database
522                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
523                 if (retModify.isSuccess()) {
524                     i++;
525                 } else {
526                     break;
527                 }
528             }
529             // Check if uncompleted modify
530             if (i < size) {
531                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
532                 // Restore original entries
533                 int j = 0;
534                 while (j < i) {
535                     log.info("Attempting to restore initial entries");
536                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
537                     if (retExt.isSuccess()) {
538                         j++;
539                     } else {
540                         break;
541                     }
542                 }
543                 // Fatal error, recovery failed
544                 if (j < i) {
545                     String msg = "Flow recovery failed ! Unrecoverable Error";
546                     log.error(msg);
547                     return new Status(StatusCode.INTERNALERROR, msg);
548                 }
549             }
550             succeeded = retModify;
551         }
552         /*
553          * The first successful status response will be returned. For the
554          * asynchronous call, we can discard the container flow complication for
555          * now and assume we will always deal with one flow only per request
556          */
557         return succeeded;
558     }
559
560     /**
561      * This is the function that modifies the final container flows merged
562      * entries on the network node and update the database. It expects that all
563      * the validity checks are passed
564      *
565      * @param currentEntries
566      * @param newEntries
567      * @param async
568      *            the flag indicating if this is a asynchronous request
569      * @return the status of this request. In case of asynchronous call, it will
570      *         contain the unique id assigned to this request
571      */
572     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
573         FlowEntryDistributionOrderFutureTask futureStatus =
574                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
575         if (futureStatus != null) {
576             Status retStatus = new Status(StatusCode.UNDEFINED);
577             try {
578                 retStatus = futureStatus.get();
579                 if (retStatus.getCode()
580                         .equals(StatusCode.TIMEOUT)) {
581                     // A timeout happened, lets cleanup the workMonitor
582                     workMonitor.remove(futureStatus.getOrder());
583                 }
584             } catch (InterruptedException e) {
585                 log.error("", e);
586             } catch (ExecutionException e) {
587                 log.error("", e);
588             }
589             return retStatus;
590         } else {
591             // Modify the flow on the network node
592             Status status = async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
593                     .getFlow(), newEntries.getInstall()
594                     .getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries.getInstall()
595                     .getFlow(), newEntries.getInstall()
596                     .getFlow());
597
598             if (!status.isSuccess()) {
599                 log.trace("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
600                         status.getDescription());
601                 return status;
602             }
603
604             log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
605
606             // Update DB
607             newEntries.setRequestId(status.getRequestId());
608             updateSwViews(currentEntries, false);
609             updateSwViews(newEntries, true);
610
611             return status;
612         }
613     }
614
615     /**
616      * Remove a flow entry. If the entry is not present in the software view
617      * (entry or node not present), it return successfully
618      *
619      * @param flowEntry
620      *            the flow entry to remove
621      * @param async
622      *            the flag indicating if this is a asynchronous request
623      * @return the status of this request. In case of asynchronous call, it will
624      *         contain the unique id assigned to this request
625      */
626     private Status removeEntry(FlowEntry flowEntry, boolean async) {
627         Status error = new Status(null, null);
628
629         // Sanity Check
630         if (flowEntry == null || flowEntry.getNode() == null || flowEntry.getFlow() == null) {
631             String logMsg = INVALID_FLOW_ENTRY + ": {}";
632             log.warn(logMsg, flowEntry);
633             return new Status(StatusCode.NOTACCEPTABLE, INVALID_FLOW_ENTRY);
634         }
635
636         // Derive the container flows merged installed entries
637         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
638
639         Status succeeded = null;
640         boolean atLeastOneRemoved = false;
641         for (FlowEntryInstall entry : installedList) {
642             if (!installedSwView.containsKey(entry)) {
643                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
644                 log.debug(logMsg, flowEntry);
645                 if (installedList.size() == 1) {
646                     // If we had only one entry to remove, we are done
647                     return new Status(StatusCode.SUCCESS);
648                 } else {
649                     continue;
650                 }
651             }
652
653             // Remove and update DB
654             Status ret = removeEntryInternal(entry, async);
655
656             if (!ret.isSuccess()) {
657                 error = ret;
658                 log.trace("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
659                 if (installedList.size() == 1) {
660                     // If we had only one entry to remove, this is fatal failure
661                     return error;
662                 }
663             } else {
664                 succeeded = ret;
665                 atLeastOneRemoved = true;
666             }
667         }
668
669         /*
670          * No worries if full removal failed. Consistency checker will take care
671          * of removing the stale entries later, or adjusting the software
672          * database if not in sync with hardware
673          */
674         return (atLeastOneRemoved) ? succeeded : error;
675     }
676
677     /**
678      * This is the function that removes the final container flows merged entry
679      * from the network node and update the database. It expects that all the
680      * validity checks are passed
681      *
682      * @param entry
683      *            the flow entry to remove
684      * @param async
685      *            the flag indicating if this is a asynchronous request
686      * @return the status of this request. In case of asynchronous call, it will
687      *         contain the unique id assigned to this request
688      */
689     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
690         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
691         if (futureStatus != null) {
692             Status retStatus = new Status(StatusCode.UNDEFINED);
693             try {
694                 retStatus = futureStatus.get();
695                 if (retStatus.getCode()
696                         .equals(StatusCode.TIMEOUT)) {
697                     // A timeout happened, lets cleanup the workMonitor
698                     workMonitor.remove(futureStatus.getOrder());
699                 }
700             } catch (InterruptedException e) {
701                 log.error("", e);
702             } catch (ExecutionException e) {
703                 log.error("", e);
704             }
705             return retStatus;
706         } else {
707             // Mark the entry to be deleted (for CC just in case we fail)
708             entry.toBeDeleted();
709
710             // Remove from node
711             Status status = async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall()
712                     .getFlow()) : programmer.removeFlow(entry.getNode(), entry.getInstall()
713                     .getFlow());
714
715             if (!status.isSuccess()) {
716                 log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", entry.getInstall(),
717                         status.getDescription());
718                 return status;
719             }
720             log.trace("Removed  {}", entry.getInstall());
721
722             // Update DB
723             updateSwViews(entry, false);
724
725             return status;
726         }
727     }
728
729     /**
730      * This is the function that installs the final container flow merged entry
731      * on the network node and updates the database. It expects that all the
732      * validity and conflict checks are passed. That means it does not check
733      * whether this flow would conflict or overwrite an existing one.
734      *
735      * @param entry
736      *            the flow entry to install
737      * @param async
738      *            the flag indicating if this is a asynchronous request
739      * @return the status of this request. In case of asynchronous call, it will
740      *         contain the unique id assigned to this request
741      */
742     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
743         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
744         if (futureStatus != null) {
745             Status retStatus = new Status(StatusCode.UNDEFINED);
746             try {
747                 retStatus = futureStatus.get();
748                 if (retStatus.getCode()
749                         .equals(StatusCode.TIMEOUT)) {
750                     // A timeout happened, lets cleanup the workMonitor
751                     workMonitor.remove(futureStatus.getOrder());
752                 }
753             } catch (InterruptedException e) {
754                 log.error("", e);
755             } catch (ExecutionException e) {
756                 log.error("", e);
757             }
758             return retStatus;
759         } else {
760             // Install the flow on the network node
761             Status status = async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall()
762                     .getFlow()) : programmer.addFlow(entry.getNode(), entry.getInstall()
763                     .getFlow());
764
765             if (!status.isSuccess()) {
766                 log.trace("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
767                         status.getDescription());
768                 return status;
769             }
770
771             log.trace("Added    {}", entry.getInstall());
772
773             // Update DB
774             entry.setRequestId(status.getRequestId());
775             updateSwViews(entry, true);
776
777             return status;
778         }
779     }
780
781     /**
782      * Returns true if the flow conflicts with all the container's flows. This
783      * means that if the function returns true, the passed flow entry is
784      * congruent with at least one container flow, hence it is good to be
785      * installed on this container.
786      *
787      * @param flowEntry
788      * @return true if flow conflicts with all the container flows, false
789      *         otherwise
790      */
791     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
792         List<ContainerFlow> cFlowList = container.getContainerFlows();
793
794         // Validity check and avoid unnecessary computation
795         // Also takes care of default container where no container flows are
796         // present
797         if (cFlowList == null || cFlowList.isEmpty()) {
798             return false;
799         }
800
801         for (ContainerFlow cFlow : cFlowList) {
802             if (cFlow.allowsFlow(flowEntry.getFlow())) {
803                 // Entry is allowed by at least one container flow: good to go
804                 return false;
805             }
806         }
807         return true;
808     }
809
810     private ConcurrentMap.Entry<Integer, FlowConfig> getStaticFlowEntry(String name, Node node) {
811         for (ConcurrentMap.Entry<Integer, FlowConfig> flowEntry : staticFlows.entrySet()) {
812             FlowConfig flowConfig = flowEntry.getValue();
813             if (flowConfig.isByNameAndNodeIdEqual(name, node)) {
814                 return flowEntry;
815             }
816         }
817         return null;
818     }
819
820     private void updateIndexDatabase(FlowEntryInstall entry, boolean add) {
821         // Update node indexed flow database
822         updateNodeFlowsDB(entry, add);
823
824         // Update group indexed flow database
825         updateGroupFlowsDB(entry, add);
826     }
827
828     /*
829      * Update the node mapped flows database
830      */
831     private void updateSwViews(FlowEntryInstall flowEntries, boolean add) {
832         if (add) {
833             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
834             installedSwView.put(flowEntries, flowEntries);
835         } else {
836             originalSwView.remove(flowEntries.getOriginal());
837             installedSwView.remove(flowEntries);
838         }
839     }
840
841     /*
842      * Update the node mapped flows database
843      */
844     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
845         Node node = flowEntries.getNode();
846
847         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
848         if (nodeIndeces == null) {
849             if (!add) {
850                 return;
851             } else {
852                 nodeIndeces = new ArrayList<FlowEntryInstall>();
853             }
854         }
855
856         if (add) {
857             nodeIndeces.add(flowEntries);
858         } else {
859             nodeIndeces.remove(flowEntries);
860         }
861
862         // Update cache across cluster
863         if (nodeIndeces.isEmpty()) {
864             this.nodeFlows.remove(node);
865         } else {
866             this.nodeFlows.put(node, nodeIndeces);
867         }
868     }
869
870     /*
871      * Update the group name mapped flows database
872      */
873     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
874         String groupName = flowEntries.getGroupName();
875
876         // Flow may not be part of a group
877         if (groupName == null) {
878             return;
879         }
880
881         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
882         if (indices == null) {
883             if (!add) {
884                 return;
885             } else {
886                 indices = new ArrayList<FlowEntryInstall>();
887             }
888         }
889
890         if (add) {
891             indices.add(flowEntries);
892         } else {
893             indices.remove(flowEntries);
894         }
895
896         // Update cache across cluster
897         if (indices.isEmpty()) {
898             this.groupFlows.remove(groupName);
899         } else {
900             this.groupFlows.put(groupName, indices);
901         }
902     }
903
904     /**
905      * Remove a flow entry that has been added previously First checks if the
906      * entry is effectively present in the local database
907      */
908     @SuppressWarnings("unused")
909     private Status removeEntry(Node node, String flowName) {
910         FlowEntryInstall target = null;
911
912         // Find in database
913         for (FlowEntryInstall entry : installedSwView.values()) {
914             if (entry.equalsByNodeAndName(node, flowName)) {
915                 target = entry;
916                 break;
917             }
918         }
919
920         // If it is not there, stop any further processing
921         if (target == null) {
922             return new Status(StatusCode.SUCCESS, "Entry is not present");
923         }
924
925         // Remove from node
926         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
927
928         // Update DB
929         if (status.isSuccess()) {
930             updateSwViews(target, false);
931         } else {
932             // log the error
933             log.trace("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
934                     status.getDescription());
935         }
936
937         return status;
938     }
939
940     @Override
941     public Status installFlowEntry(FlowEntry flowEntry) {
942         Status status;
943         if (isContainerModeAllowed(flowEntry)) {
944             status = addEntry(flowEntry, false);
945         } else {
946             String msg = "Controller in container mode: Install Refused";
947             String logMsg = msg + ": {}";
948             status = new Status(StatusCode.NOTACCEPTABLE, msg);
949             log.warn(logMsg, flowEntry);
950         }
951         return status;
952     }
953
954     @Override
955     public Status installFlowEntryAsync(FlowEntry flowEntry) {
956         Status status;
957         if (isContainerModeAllowed(flowEntry)) {
958             status = addEntry(flowEntry, true);
959         } else {
960             String msg = "Controller in container mode: Install Refused";
961             status = new Status(StatusCode.NOTACCEPTABLE, msg);
962             log.warn(msg);
963         }
964         return status;
965     }
966
967     @Override
968     public Status uninstallFlowEntry(FlowEntry flowEntry) {
969         Status status;
970         if (isContainerModeAllowed(flowEntry)) {
971             status = removeEntry(flowEntry, false);
972         } else {
973             String msg = "Controller in container mode: Uninstall Refused";
974             String logMsg = msg + ": {}";
975             status = new Status(StatusCode.NOTACCEPTABLE, msg);
976             log.warn(logMsg, flowEntry);
977         }
978         return status;
979     }
980
981     @Override
982     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
983         Status status;
984         if (isContainerModeAllowed(flowEntry)) {
985             status = removeEntry(flowEntry, true);
986         } else {
987             String msg = "Controller in container mode: Uninstall Refused";
988             status = new Status(StatusCode.NOTACCEPTABLE, msg);
989             log.warn(msg);
990         }
991         return status;
992     }
993
994     @Override
995     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
996         Status status = null;
997         if (isContainerModeAllowed(currentFlowEntry)) {
998             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
999         } else {
1000             String msg = "Controller in container mode: Modify Refused";
1001             String logMsg = msg + ": {}";
1002             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1003             log.warn(logMsg, newFlowEntry);
1004         }
1005         return status;
1006     }
1007
1008     @Override
1009     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
1010         Status status = null;
1011         if (isContainerModeAllowed(currentFlowEntry)) {
1012             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
1013         } else {
1014             String msg = "Controller in container mode: Modify Refused";
1015             status = new Status(StatusCode.NOTACCEPTABLE, msg);
1016             log.warn(msg);
1017         }
1018         return status;
1019     }
1020
1021     /**
1022      * Returns whether the specified flow entry is allowed to be
1023      * installed/removed/modified based on the current container mode status.
1024      * This call always returns true in the container instance of forwarding
1025      * rules manager. It is meant for the global instance only (default
1026      * container) of forwarding rules manager. Idea is that for assuring
1027      * container isolation of traffic, flow installation in default container is
1028      * blocked when in container mode (containers are present). The only flows
1029      * that are allowed in container mode in the default container are the
1030      * proactive flows, the ones automatically installed on the network node
1031      * which forwarding mode has been configured to "proactive". These flows are
1032      * needed by controller to discover the nodes topology and to discover the
1033      * attached hosts for some SDN switches.
1034      *
1035      * @param flowEntry
1036      *            The flow entry to be installed/removed/modified
1037      * @return true if not in container mode or if flowEntry is internally
1038      *         generated
1039      */
1040     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1041         return (!inContainerMode) ? true : flowEntry.isInternal();
1042     }
1043
1044     @Override
1045     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1046         /*
1047          * Run a check on the original entries to decide whether to go with a
1048          * add or modify method. A loose check means only check against the
1049          * original flow entry requests and not against the installed flow
1050          * entries which are the result of the original entry merged with the
1051          * container flow(s) (if any). The modifyFlowEntry method in presence of
1052          * conflicts with the Container flows (if any) would revert back to a
1053          * delete + add pattern
1054          */
1055         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1056
1057         if (currentFlowEntry != null) {
1058             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1059         } else {
1060             return installFlowEntry(newFlowEntry);
1061         }
1062     }
1063
1064     @Override
1065     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1066         /*
1067          * Run a check on the original entries to decide whether to go with a
1068          * add or modify method. A loose check means only check against the
1069          * original flow entry requests and not against the installed flow
1070          * entries which are the result of the original entry merged with the
1071          * container flow(s) (if any). The modifyFlowEntry method in presence of
1072          * conflicts with the Container flows (if any) would revert back to a
1073          * delete + add pattern
1074          */
1075         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1076
1077         if (currentFlowEntry != null) {
1078             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1079         } else {
1080             return installFlowEntryAsync(newFlowEntry);
1081         }
1082     }
1083
1084     @Override
1085     public Status uninstallFlowEntryGroup(String groupName) {
1086         if (groupName == null || groupName.isEmpty()) {
1087             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1088         }
1089         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1090             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1091         }
1092         if (inContainerMode) {
1093             String msg = "Controller in container mode: Group Uninstall Refused";
1094             String logMsg = msg + ": {}";
1095             log.warn(logMsg, groupName);
1096             return new Status(StatusCode.NOTACCEPTABLE, msg);
1097         }
1098         int toBeRemoved = 0;
1099         String error = "";
1100         if (groupFlows.containsKey(groupName)) {
1101             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1102             toBeRemoved = list.size();
1103             for (FlowEntryInstall entry : list) {
1104                 Status status = this.removeEntry(entry.getOriginal(), false);
1105                 if (status.isSuccess()) {
1106                     toBeRemoved -= 1;
1107                 } else {
1108                     error = status.getDescription();
1109                 }
1110             }
1111         }
1112         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1113                 "Not all the flows were removed: " + error);
1114     }
1115
1116     @Override
1117     public Status uninstallFlowEntryGroupAsync(String groupName) {
1118         if (groupName == null || groupName.isEmpty()) {
1119             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1120         }
1121         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1122             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1123         }
1124         if (inContainerMode) {
1125             String msg = "Controller in container mode: Group Uninstall Refused";
1126             String logMsg = msg + ": {}";
1127             log.warn(logMsg, groupName);
1128             return new Status(StatusCode.NOTACCEPTABLE, msg);
1129         }
1130         if (groupFlows.containsKey(groupName)) {
1131             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1132             for (FlowEntryInstall entry : list) {
1133                 this.removeEntry(entry.getOriginal(), true);
1134             }
1135         }
1136         return new Status(StatusCode.SUCCESS);
1137     }
1138
1139     @Override
1140     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1141         return entryConflictsWithContainerFlows(flowEntry);
1142     }
1143
1144     /**
1145      * Updates all installed flows because the container flow got updated This
1146      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1147      * Reinstall of all flows This is needed because a new container flows
1148      * merged flow may conflict with an existing old container flows merged flow
1149      * on the network node
1150      */
1151     protected void updateFlowsContainerFlow() {
1152         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1153         // First remove all installed entries
1154         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1155             FlowEntryInstall current = entry.getValue();
1156             // Store the original entry
1157             toReInstall.add(current.getOriginal());
1158             // Remove the old couples. No validity checks to be run, use the
1159             // internal remove
1160             this.removeEntryInternal(current, false);
1161         }
1162         // Then reinstall the original entries
1163         for (FlowEntry entry : toReInstall) {
1164             // Reinstall the original flow entries, via the regular path: new
1165             // cFlow merge + validations
1166             this.installFlowEntry(entry);
1167         }
1168     }
1169
1170     private void nonClusterObjectCreate() {
1171         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1172         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1173         TSPolicies = new ConcurrentHashMap<String, Object>();
1174         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1175         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1176         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1177         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1178         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1179     }
1180
1181     private void registerWithOSGIConsole() {
1182         BundleContext bundleContext = FrameworkUtil.getBundle(this.getClass()).getBundleContext();
1183         bundleContext.registerService(CommandProvider.class.getName(), this, null);
1184     }
1185
1186     @Override
1187     public void setTSPolicyData(String policyname, Object o, boolean add) {
1188
1189         if (add) {
1190             /* Check if this policy already exists */
1191             if (!(TSPolicies.containsKey(policyname))) {
1192                 TSPolicies.put(policyname, o);
1193             }
1194         } else {
1195             TSPolicies.remove(policyname);
1196         }
1197         if (frmAware != null) {
1198             synchronized (frmAware) {
1199                 for (IForwardingRulesManagerAware frma : frmAware) {
1200                     try {
1201                         frma.policyUpdate(policyname, add);
1202                     } catch (Exception e) {
1203                         log.warn("Exception on callback", e);
1204                     }
1205                 }
1206             }
1207         }
1208     }
1209
1210     @Override
1211     public Map<String, Object> getTSPolicyData() {
1212         return TSPolicies;
1213     }
1214
1215     @Override
1216     public Object getTSPolicyData(String policyName) {
1217         if (TSPolicies.containsKey(policyName)) {
1218             return TSPolicies.get(policyName);
1219         } else {
1220             return null;
1221         }
1222     }
1223
1224     @Override
1225     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1226         List<FlowEntry> list = new ArrayList<FlowEntry>();
1227         if (policyName != null && !policyName.trim().isEmpty()) {
1228             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1229                 if (policyName.equals(entry.getKey().getGroupName())) {
1230                     list.add(entry.getKey().clone());
1231                 }
1232             }
1233         }
1234         return list;
1235     }
1236
1237     @Override
1238     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1239         List<FlowEntry> list = new ArrayList<FlowEntry>();
1240         if (policyName != null && !policyName.trim().isEmpty()) {
1241             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1242                 if (policyName.equals(entry.getKey().getGroupName())) {
1243                     list.add(entry.getKey().getInstall().clone());
1244                 }
1245             }
1246         }
1247         return list;
1248     }
1249
1250     @Override
1251     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1252
1253         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1254             if (flow.getFlowName().equals(flowName)) {
1255                 FlowEntry currentFlowEntry = flow.getOriginal();
1256                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1257                 for (NodeConnector dstPort : portList) {
1258                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1259                 }
1260                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1261                 if (error.isSuccess()) {
1262                     log.info("Ports {} added to FlowEntry {}", portList, flowName);
1263                 } else {
1264                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1265                             currentFlowEntry.toString(), error.getDescription());
1266                 }
1267                 return;
1268             }
1269         }
1270         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1271     }
1272
1273     @Override
1274     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1275         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1276             FlowEntryInstall flow = this.installedSwView.get(index);
1277             if (flow.getFlowName().equals(flowName)) {
1278                 FlowEntry currentFlowEntry = flow.getOriginal();
1279                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1280                 for (NodeConnector dstPort : portList) {
1281                     Action action = new Output(dstPort);
1282                     newFlowEntry.getFlow().removeAction(action);
1283                 }
1284                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1285                 if (status.isSuccess()) {
1286                     log.info("Ports {} removed from FlowEntry {}", portList, flowName);
1287                 } else {
1288                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1289                             currentFlowEntry.toString(), status.getDescription());
1290                 }
1291                 return;
1292             }
1293         }
1294         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1295     }
1296
1297     /*
1298      * This function assumes the target flow has only one output port
1299      */
1300     @Override
1301     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1302         FlowEntry currentFlowEntry = null;
1303         FlowEntry newFlowEntry = null;
1304
1305         // Find the flow
1306         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1307             FlowEntryInstall flow = this.installedSwView.get(index);
1308             if (flow.getFlowName().equals(flowName)) {
1309                 currentFlowEntry = flow.getOriginal();
1310                 break;
1311             }
1312         }
1313         if (currentFlowEntry == null) {
1314             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1315             return;
1316         }
1317
1318         // Create a flow copy with the new output port
1319         newFlowEntry = currentFlowEntry.clone();
1320         Action target = null;
1321         for (Action action : newFlowEntry.getFlow().getActions()) {
1322             if (action.getType() == ActionType.OUTPUT) {
1323                 target = action;
1324                 break;
1325             }
1326         }
1327         newFlowEntry.getFlow().removeAction(target);
1328         newFlowEntry.getFlow().addAction(new Output(outPort));
1329
1330         // Modify on network node
1331         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1332
1333         if (status.isSuccess()) {
1334             log.info("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1335         } else {
1336             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1337                     status.getDescription());
1338         }
1339         return;
1340     }
1341
1342     @Override
1343     public NodeConnector getOutputPort(Node node, String flowName) {
1344         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1345             FlowEntryInstall flow = this.installedSwView.get(index);
1346             if (flow.getFlowName().equals(flowName)) {
1347                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1348                     if (action.getType() == ActionType.OUTPUT) {
1349                         return ((Output) action).getPort();
1350                     }
1351                 }
1352             }
1353         }
1354         return null;
1355     }
1356
1357     private void cacheStartup() {
1358         allocateCaches();
1359         retrieveCaches();
1360     }
1361
1362     private void allocateCaches() {
1363         if (this.clusterContainerService == null) {
1364             log.warn("Un-initialized clusterContainerService, can't create cache");
1365             return;
1366         }
1367
1368         log.debug("Allocating caches for Container {}", container.getName());
1369
1370         try {
1371             clusterContainerService.createCache(ORIGINAL_SW_VIEW_CACHE,
1372                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1373
1374             clusterContainerService.createCache(INSTALLED_SW_VIEW_CACHE,
1375                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1376
1377             clusterContainerService.createCache("frm.inactiveFlows",
1378                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1379
1380             clusterContainerService.createCache("frm.staticFlows",
1381                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1382
1383             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1384                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1385
1386             clusterContainerService.createCache("frm.portGroupConfigs",
1387                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1388
1389             clusterContainerService.createCache("frm.portGroupData",
1390                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1391
1392             clusterContainerService.createCache("frm.TSPolicies",
1393                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1394
1395             clusterContainerService.createCache(WORK_STATUS_CACHE,
1396                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1397
1398             clusterContainerService.createCache(WORK_ORDER_CACHE,
1399                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1400
1401         } catch (CacheConfigException cce) {
1402             log.error("CacheConfigException");
1403         } catch (CacheExistException cce) {
1404             log.error("CacheExistException");
1405         }
1406     }
1407
1408     @SuppressWarnings({ "unchecked" })
1409     private void retrieveCaches() {
1410         ConcurrentMap<?, ?> map;
1411
1412         if (this.clusterContainerService == null) {
1413             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1414             nonClusterObjectCreate();
1415             return;
1416         }
1417
1418         log.debug("Retrieving Caches for Container {}", container.getName());
1419
1420         map = clusterContainerService.getCache(ORIGINAL_SW_VIEW_CACHE);
1421         if (map != null) {
1422             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1423         } else {
1424             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1425         }
1426
1427         map = clusterContainerService.getCache(INSTALLED_SW_VIEW_CACHE);
1428         if (map != null) {
1429             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1430         } else {
1431             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1432         }
1433
1434         map = clusterContainerService.getCache("frm.inactiveFlows");
1435         if (map != null) {
1436             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1437         } else {
1438             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1439         }
1440
1441         map = clusterContainerService.getCache("frm.staticFlows");
1442         if (map != null) {
1443             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1444         } else {
1445             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1446         }
1447
1448         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1449         if (map != null) {
1450             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1451         } else {
1452             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1453         }
1454
1455         map = clusterContainerService.getCache("frm.portGroupConfigs");
1456         if (map != null) {
1457             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1458         } else {
1459             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1460         }
1461
1462         map = clusterContainerService.getCache("frm.portGroupData");
1463         if (map != null) {
1464             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1465         } else {
1466             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1467         }
1468
1469         map = clusterContainerService.getCache("frm.TSPolicies");
1470         if (map != null) {
1471             TSPolicies = (ConcurrentMap<String, Object>) map;
1472         } else {
1473             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1474         }
1475
1476         map = clusterContainerService.getCache(WORK_ORDER_CACHE);
1477         if (map != null) {
1478             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1479         } else {
1480             log.error("Retrieval of " + WORK_ORDER_CACHE + " cache failed for Container {}", container.getName());
1481         }
1482
1483         map = clusterContainerService.getCache(WORK_STATUS_CACHE);
1484         if (map != null) {
1485             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1486         } else {
1487             log.error("Retrieval of " + WORK_STATUS_CACHE + " cache failed for Container {}", container.getName());
1488         }
1489     }
1490
1491     private boolean flowConfigExists(FlowConfig config) {
1492         // Flow name has to be unique on per node id basis
1493         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1494             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1495                 return true;
1496             }
1497         }
1498         return false;
1499     }
1500
1501     @Override
1502     public Status addStaticFlow(FlowConfig config) {
1503         // Configuration object validation
1504         Status status = config.validate(container);
1505         if (!status.isSuccess()) {
1506             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1507             String error = "Invalid Configuration (" + status.getDescription() + ")";
1508             config.setStatus(error);
1509             return new Status(StatusCode.BADREQUEST, error);
1510         }
1511         return addStaticFlowInternal(config, false);
1512     }
1513
1514     /**
1515      * Private method to add a static flow configuration which does not run any
1516      * validation on the passed FlowConfig object. If restore is set to true,
1517      * configuration is stored in configuration database regardless the
1518      * installation on the network node was successful. This is useful at boot
1519      * when static flows are present in startup configuration and are read
1520      * before the switches connects.
1521      *
1522      * @param config
1523      *            The static flow configuration
1524      * @param restore
1525      *            if true, the configuration is stored regardless the
1526      *            installation on the network node was successful
1527      * @return The status of this request
1528      */
1529     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1530         boolean multipleFlowPush = false;
1531         String error;
1532         Status status;
1533         config.setStatus(StatusCode.SUCCESS.toString());
1534
1535         // Presence check
1536         if (flowConfigExists(config)) {
1537             error = "Entry with this name on specified switch already exists";
1538             log.warn("Entry with this name on specified switch already exists: {}", config);
1539             config.setStatus(error);
1540             return new Status(StatusCode.CONFLICT, error);
1541         }
1542
1543         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1544             for (String portGroupName : portGroupConfigs.keySet()) {
1545                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1546                     multipleFlowPush = true;
1547                     break;
1548                 }
1549             }
1550             if (!multipleFlowPush) {
1551                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1552                 error = "Invalid Configuration (Invalid PortGroup Name)";
1553                 config.setStatus(error);
1554                 return new Status(StatusCode.BADREQUEST, error);
1555             }
1556         }
1557
1558         /*
1559          * If requested program the entry in hardware first before updating the
1560          * StaticFlow DB
1561          */
1562         if (!multipleFlowPush) {
1563             // Program hw
1564             if (config.installInHw()) {
1565                 FlowEntry entry = config.getFlowEntry();
1566                 status = this.installFlowEntry(entry);
1567                 if (!status.isSuccess()) {
1568                     config.setStatus(status.getDescription());
1569                     if (!restore) {
1570                         return status;
1571                     }
1572                 }
1573             }
1574         }
1575
1576         /*
1577          * When the control reaches this point, either of the following
1578          * conditions is true 1. This is a single entry configuration (non
1579          * PortGroup) and the hardware installation is successful 2. This is a
1580          * multiple entry configuration (PortGroup) and hardware installation is
1581          * NOT done directly on this event. 3. The User prefers to retain the
1582          * configuration in Controller and skip hardware installation.
1583          *
1584          * Hence it is safe to update the StaticFlow DB at this point.
1585          *
1586          * Note : For the case of PortGrouping, it is essential to have this DB
1587          * populated before the PortGroupListeners can query for the DB
1588          * triggered using portGroupChanged event...
1589          */
1590         Integer ordinal = staticFlowsOrdinal.get(0);
1591         staticFlowsOrdinal.put(0, ++ordinal);
1592         staticFlows.put(ordinal, config);
1593
1594         if (multipleFlowPush) {
1595             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1596             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1597             if (existingData != null) {
1598                 portGroupChanged(pgconfig, existingData, true);
1599             }
1600         }
1601         return new Status(StatusCode.SUCCESS);
1602     }
1603
1604     private void addStaticFlowsToSwitch(Node node) {
1605         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1606             FlowConfig config = entry.getValue();
1607             if (config.isPortGroupEnabled()) {
1608                 continue;
1609             }
1610             if (config.getNode().equals(node)) {
1611                 if (config.installInHw() && !config.getStatus().equals(StatusCode.SUCCESS.toString())) {
1612                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1613                     config.setStatus(status.getDescription());
1614                 }
1615             }
1616         }
1617         // Update cluster cache
1618         refreshClusterStaticFlowsStatus(node);
1619     }
1620
1621     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1622         log.trace("Updating Static Flow configs on node down: {}", node);
1623
1624         List<Integer> toRemove = new ArrayList<Integer>();
1625         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1626
1627             FlowConfig config = entry.getValue();
1628
1629             if (config.isPortGroupEnabled()) {
1630                 continue;
1631             }
1632
1633             if (config.installInHw() && config.getNode().equals(node)) {
1634                 if (config.isInternalFlow()) {
1635                     // Take note of this controller generated static flow
1636                     toRemove.add(entry.getKey());
1637                 } else {
1638                     config.setStatus(NODE_DOWN);
1639                 }
1640             }
1641         }
1642         // Remove controller generated static flows for this node
1643         for (Integer index : toRemove) {
1644             staticFlows.remove(index);
1645         }
1646         // Update cluster cache
1647         refreshClusterStaticFlowsStatus(node);
1648
1649     }
1650
1651     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1652         log.trace("Updating Static Flow configs on container mode change: {}", update);
1653
1654         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1655             FlowConfig config = entry.getValue();
1656             if (config.isPortGroupEnabled()) {
1657                 continue;
1658             }
1659             if (config.installInHw() && !config.isInternalFlow()) {
1660                 switch (update) {
1661                 case ADDED:
1662                     config.setStatus("Removed from node because in container mode");
1663                     break;
1664                 case REMOVED:
1665                     config.setStatus(StatusCode.SUCCESS.toString());
1666                     break;
1667                 default:
1668                 }
1669             }
1670         }
1671         // Update cluster cache
1672         refreshClusterStaticFlowsStatus(null);
1673     }
1674
1675     @Override
1676     public Status removeStaticFlow(FlowConfig config) {
1677         /*
1678          * No config.isInternal() check as NB does not take this path and GUI
1679          * cannot issue a delete on an internal generated flow. We need this
1680          * path to be accessible when switch mode is changed from proactive to
1681          * reactive, so that we can remove the internal generated LLDP and ARP
1682          * punt flows
1683          */
1684
1685         // Look for the target configuration entry
1686         Integer key = 0;
1687         FlowConfig target = null;
1688         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1689             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1690                 key = entry.getKey();
1691                 target = entry.getValue();
1692                 break;
1693             }
1694         }
1695         if (target == null) {
1696             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1697         }
1698
1699         // Program the network node
1700         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1701
1702         // Update configuration database if programming was successful
1703         if (status.isSuccess()) {
1704             staticFlows.remove(key);
1705         }
1706
1707         return status;
1708     }
1709
1710     @Override
1711     public Status removeStaticFlow(String name, Node node) {
1712         // Look for the target configuration entry
1713         Integer key = 0;
1714         FlowConfig target = null;
1715         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1716             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1717                 key = mapEntry.getKey();
1718                 target = mapEntry.getValue();
1719                 break;
1720             }
1721         }
1722         if (target == null) {
1723             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1724         }
1725
1726         // Validity check for api3 entry point
1727         if (target.isInternalFlow()) {
1728             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1729             String logMsg = msg + ": {}";
1730             log.warn(logMsg, name);
1731             return new Status(StatusCode.NOTACCEPTABLE, msg);
1732         }
1733
1734         if (target.isPortGroupEnabled()) {
1735             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1736             String logMsg = msg + ": {}";
1737             log.warn(logMsg, name);
1738             return new Status(StatusCode.NOTACCEPTABLE, msg);
1739         }
1740
1741         // Program the network node
1742         Status status = this.removeEntry(target.getFlowEntry(), false);
1743
1744         // Update configuration database if programming was successful
1745         if (status.isSuccess()) {
1746             staticFlows.remove(key);
1747         }
1748
1749         return status;
1750     }
1751
1752     @Override
1753     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1754         // Validity check for api3 entry point
1755         if (newFlowConfig.isInternalFlow()) {
1756             String msg = "Invalid operation: Controller generated flow cannot be modified";
1757             String logMsg = msg + ": {}";
1758             log.warn(logMsg, newFlowConfig);
1759             return new Status(StatusCode.NOTACCEPTABLE, msg);
1760         }
1761
1762         // Validity Check
1763         Status status = newFlowConfig.validate(container);
1764         if (!status.isSuccess()) {
1765             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1766             newFlowConfig.setStatus(msg);
1767             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1768             return new Status(StatusCode.BADREQUEST, msg);
1769         }
1770
1771         FlowConfig oldFlowConfig = null;
1772         Integer index = null;
1773         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1774             FlowConfig entry = mapEntry.getValue();
1775             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1776                 oldFlowConfig = entry;
1777                 index = mapEntry.getKey();
1778                 break;
1779             }
1780         }
1781
1782         if (oldFlowConfig == null) {
1783             String msg = "Attempt to modify a non existing static flow";
1784             String logMsg = msg + ": {}";
1785             log.warn(logMsg, newFlowConfig);
1786             return new Status(StatusCode.NOTFOUND, msg);
1787         }
1788
1789         // Do not attempt to reinstall the flow, warn user
1790         if (newFlowConfig.equals(oldFlowConfig)) {
1791             String msg = "No modification detected";
1792             log.info("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1793             return new Status(StatusCode.SUCCESS, msg);
1794         }
1795
1796         // If flow is installed, program the network node
1797         status = new Status(StatusCode.SUCCESS, "Saved in config");
1798         if (oldFlowConfig.installInHw()) {
1799             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1800         }
1801
1802         // Update configuration database if programming was successful
1803         if (status.isSuccess()) {
1804             newFlowConfig.setStatus(status.getDescription());
1805             staticFlows.put(index, newFlowConfig);
1806         }
1807
1808         return status;
1809     }
1810
1811     @Override
1812     public Status toggleStaticFlowStatus(String name, Node node) {
1813         return toggleStaticFlowStatus(getStaticFlow(name, node));
1814     }
1815
1816     @Override
1817     public Status toggleStaticFlowStatus(FlowConfig config) {
1818         if (config == null) {
1819             String msg = "Invalid request: null flow config";
1820             log.warn(msg);
1821             return new Status(StatusCode.BADREQUEST, msg);
1822         }
1823         // Validity check for api3 entry point
1824         if (config.isInternalFlow()) {
1825             String msg = "Invalid operation: Controller generated flow cannot be modified";
1826             String logMsg = msg + ": {}";
1827             log.warn(logMsg, config);
1828             return new Status(StatusCode.NOTACCEPTABLE, msg);
1829         }
1830
1831         // Find the config entry
1832         Integer key = 0;
1833         FlowConfig target = null;
1834         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1835             FlowConfig conf = entry.getValue();
1836             if (conf.isByNameAndNodeIdEqual(config)) {
1837                 key = entry.getKey();
1838                 target = conf;
1839                 break;
1840             }
1841         }
1842         if (target != null) {
1843             Status status = target.validate(container);
1844             if (!status.isSuccess()) {
1845                 log.warn(status.getDescription());
1846                 return status;
1847             }
1848             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1849                                     .installFlowEntry(target.getFlowEntry());
1850             if (status.isSuccess()) {
1851                 // Update Configuration database
1852                 target.setStatus(StatusCode.SUCCESS.toString());
1853                 target.toggleInstallation();
1854                 staticFlows.put(key, target);
1855             }
1856             return status;
1857         }
1858
1859         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1860     }
1861
1862     /**
1863      * Reinsert all static flows entries in the cache to force cache updates in
1864      * the cluster. This is useful when only some parameters were changed in the
1865      * entries, like the status.
1866      *
1867      * @param node
1868      *            The node for which the static flow configurations have to be
1869      *            refreshed. If null, all nodes static flows will be refreshed.
1870      */
1871     private void refreshClusterStaticFlowsStatus(Node node) {
1872         // Refresh cluster cache
1873         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1874             if (node == null || entry.getValue().getNode().equals(node)) {
1875                 staticFlows.put(entry.getKey(), entry.getValue());
1876             }
1877         }
1878     }
1879
1880     /**
1881      * Uninstall all the non-internal Flow Entries present in the software view.
1882      * If requested, a copy of each original flow entry will be stored in the
1883      * inactive list so that it can be re-applied when needed (This is typically
1884      * the case when running in the default container and controller moved to
1885      * container mode) NOTE WELL: The routine as long as does a bulk change will
1886      * operate only on the entries for nodes locally attached so to avoid
1887      * redundant operations initiated by multiple nodes
1888      *
1889      * @param preserveFlowEntries
1890      *            if true, a copy of each original entry is stored in the
1891      *            inactive list
1892      */
1893     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1894         log.info("Uninstalling all non-internal flows");
1895
1896         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1897
1898         // Store entries / create target list
1899         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1900             FlowEntryInstall flowEntries = mapEntry.getValue();
1901             // Skip internal generated static flows
1902             if (!flowEntries.isInternal()) {
1903                 toRemove.add(flowEntries);
1904                 // Store the original entries if requested
1905                 if (preserveFlowEntries) {
1906                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1907                 }
1908             }
1909         }
1910
1911         // Now remove the entries
1912         for (FlowEntryInstall flowEntryHw : toRemove) {
1913             Node n = flowEntryHw.getNode();
1914             if (n != null && connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
1915                 Status status = this.removeEntryInternal(flowEntryHw, false);
1916                 if (!status.isSuccess()) {
1917                     log.trace("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1918                 }
1919             } else {
1920                 log.debug("Not removing entry {} because not connected locally, the remote guy will do it's job",
1921                         flowEntryHw);
1922             }
1923         }
1924     }
1925
1926     /**
1927      * Re-install all the Flow Entries present in the inactive list The inactive
1928      * list will be empty at the end of this call This function is called on the
1929      * default container instance of FRM only when the last container is deleted
1930      */
1931     private void reinstallAllFlowEntries() {
1932         log.info("Reinstalling all inactive flows");
1933
1934         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1935             this.addEntry(flowEntry, false);
1936         }
1937
1938         // Empty inactive list in any case
1939         inactiveFlows.clear();
1940     }
1941
1942     @Override
1943     public List<FlowConfig> getStaticFlows() {
1944         return getStaticFlowsOrderedList(staticFlows, staticFlowsOrdinal.get(0).intValue());
1945     }
1946
1947     // TODO: need to come out with a better algorithm for maintaining the order
1948     // of the configuration entries
1949     // with actual one, index associated to deleted entries cannot be reused and
1950     // map grows...
1951     private List<FlowConfig> getStaticFlowsOrderedList(ConcurrentMap<Integer, FlowConfig> flowMap, int maxKey) {
1952         List<FlowConfig> orderedList = new ArrayList<FlowConfig>();
1953         for (int i = 0; i <= maxKey; i++) {
1954             FlowConfig entry = flowMap.get(i);
1955             if (entry != null) {
1956                 orderedList.add(entry);
1957             }
1958         }
1959         return orderedList;
1960     }
1961
1962     @Override
1963     public FlowConfig getStaticFlow(String name, Node node) {
1964         ConcurrentMap.Entry<Integer, FlowConfig> entry = getStaticFlowEntry(name, node);
1965         if(entry != null) {
1966             return entry.getValue();
1967         }
1968         return null;
1969     }
1970
1971     @Override
1972     public List<FlowConfig> getStaticFlows(Node node) {
1973         List<FlowConfig> list = new ArrayList<FlowConfig>();
1974         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1975             if (entry.getValue().onNode(node)) {
1976                 list.add(entry.getValue());
1977             }
1978         }
1979         return list;
1980     }
1981
1982     @Override
1983     public List<String> getStaticFlowNamesForNode(Node node) {
1984         List<String> list = new ArrayList<String>();
1985         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1986             if (entry.getValue().onNode(node)) {
1987                 list.add(entry.getValue().getName());
1988             }
1989         }
1990         return list;
1991     }
1992
1993     @Override
1994     public List<Node> getListNodeWithConfiguredFlows() {
1995         Set<Node> set = new HashSet<Node>();
1996         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1997             set.add(entry.getValue().getNode());
1998         }
1999         return new ArrayList<Node>(set);
2000     }
2001
2002     @SuppressWarnings("unchecked")
2003     private void loadFlowConfiguration() {
2004         ObjectReader objReader = new ObjectReader();
2005         ConcurrentMap<Integer, FlowConfig> confList = (ConcurrentMap<Integer, FlowConfig>) objReader.read(this,
2006                 frmFileName);
2007
2008         ConcurrentMap<String, PortGroupConfig> pgConfig = (ConcurrentMap<String, PortGroupConfig>) objReader.read(this,
2009                 portGroupFileName);
2010
2011         if (pgConfig != null) {
2012             for (ConcurrentMap.Entry<String, PortGroupConfig> entry : pgConfig.entrySet()) {
2013                 addPortGroupConfig(entry.getKey(), entry.getValue().getMatchString(), true);
2014             }
2015         }
2016
2017         if (confList == null) {
2018             return;
2019         }
2020
2021         int maxKey = 0;
2022         for (Integer key : confList.keySet()) {
2023             if (key.intValue() > maxKey) {
2024                 maxKey = key.intValue();
2025             }
2026         }
2027
2028         for (FlowConfig conf : getStaticFlowsOrderedList(confList, maxKey)) {
2029             addStaticFlowInternal(conf, true);
2030         }
2031     }
2032
2033     @Override
2034     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2035         return ois.readObject();
2036     }
2037
2038     @Override
2039     public Status saveConfig() {
2040         return saveConfigInternal();
2041     }
2042
2043     private Status saveConfigInternal() {
2044         ObjectWriter objWriter = new ObjectWriter();
2045         ConcurrentMap<Integer, FlowConfig> nonDynamicFlows = new ConcurrentHashMap<Integer, FlowConfig>();
2046         for (Integer ordinal : staticFlows.keySet()) {
2047             FlowConfig config = staticFlows.get(ordinal);
2048             // Do not save dynamic and controller generated static flows
2049             if (config.isDynamic() || config.isInternalFlow()) {
2050                 continue;
2051             }
2052             nonDynamicFlows.put(ordinal, config);
2053         }
2054         objWriter.write(nonDynamicFlows, frmFileName);
2055         objWriter.write(new ConcurrentHashMap<String, PortGroupConfig>(portGroupConfigs), portGroupFileName);
2056         return new Status(StatusCode.SUCCESS, null);
2057     }
2058
2059     @Override
2060     public void subnetNotify(Subnet sub, boolean add) {
2061     }
2062
2063     /**
2064      * (non-Javadoc)
2065      *
2066      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2067      *      boolean)
2068      *
2069      *      This method can be called from within the OSGi framework context,
2070      *      given the programming operation can take sometime, it not good
2071      *      pratice to have in it's context operations that can take time,
2072      *      hence moving off to a different thread for async processing.
2073      */
2074     private ExecutorService executor;
2075     @Override
2076     public void modeChangeNotify(final Node node, final boolean proactive) {
2077         Callable<Status> modeChangeCallable = new Callable<Status>() {
2078             @Override
2079             public Status call() throws Exception {
2080                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2081
2082                 List<String> puntAction = new ArrayList<String>();
2083                 puntAction.add(ActionType.CONTROLLER.toString());
2084
2085                 FlowConfig allowARP = new FlowConfig();
2086                 allowARP.setInstallInHw(true);
2087                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2088                 allowARP.setPriority("1");
2089                 allowARP.setNode(node);
2090                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2091                         .toUpperCase());
2092                 allowARP.setActions(puntAction);
2093                 defaultConfigs.add(allowARP);
2094
2095                 FlowConfig allowLLDP = new FlowConfig();
2096                 allowLLDP.setInstallInHw(true);
2097                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2098                 allowLLDP.setPriority("1");
2099                 allowLLDP.setNode(node);
2100                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2101                         .toUpperCase());
2102                 allowLLDP.setActions(puntAction);
2103                 defaultConfigs.add(allowLLDP);
2104
2105                 List<String> dropAction = new ArrayList<String>();
2106                 dropAction.add(ActionType.DROP.toString());
2107
2108                 FlowConfig dropAllConfig = new FlowConfig();
2109                 dropAllConfig.setInstallInHw(true);
2110                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2111                         + FlowConfig.INTERNALSTATICFLOWEND);
2112                 dropAllConfig.setPriority("0");
2113                 dropAllConfig.setNode(node);
2114                 dropAllConfig.setActions(dropAction);
2115                 defaultConfigs.add(dropAllConfig);
2116
2117                 log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2118                 for (FlowConfig fc : defaultConfigs) {
2119                     Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2120                     if (status.isSuccess()) {
2121                         log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2122                     } else {
2123                         log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2124                                 fc.getName());
2125                     }
2126                 }
2127                 return new Status(StatusCode.SUCCESS);
2128             }
2129         };
2130
2131         /*
2132          * Execute the work outside the caller context, this could be an
2133          * expensive operation and we don't want to block the caller for it.
2134          */
2135         this.executor.submit(modeChangeCallable);
2136     }
2137
2138     /**
2139      * Remove from the databases all the flows installed on the node
2140      *
2141      * @param node
2142      */
2143     private void cleanDatabaseForNode(Node node) {
2144         log.info("Cleaning Flow database for Node {}", node);
2145         if (nodeFlows.containsKey(node)) {
2146             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2147
2148             for (FlowEntryInstall entry : toRemove) {
2149                 updateSwViews(entry, false);
2150             }
2151         }
2152     }
2153
2154     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2155         if (nc == null) {
2156             return false;
2157         }
2158
2159         Match match = flow.getMatch();
2160         if (match.isPresent(MatchType.IN_PORT)) {
2161             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2162             if (matchPort.equals(nc)) {
2163                 return true;
2164             }
2165         }
2166         List<Action> actionsList = flow.getActions();
2167         if (actionsList != null) {
2168             for (Action action : actionsList) {
2169                 if (action instanceof Output) {
2170                     NodeConnector actionPort = ((Output) action).getPort();
2171                     if (actionPort.equals(nc)) {
2172                         return true;
2173                     }
2174                 }
2175             }
2176         }
2177         return false;
2178     }
2179
2180     @Override
2181     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2182         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2183     }
2184
2185     @Override
2186     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2187         boolean updateStaticFlowCluster = false;
2188
2189         switch (type) {
2190         case ADDED:
2191             break;
2192         case CHANGED:
2193             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2194             if (config != null) {
2195                 switch (config.getValue()) {
2196                 case Config.ADMIN_DOWN:
2197                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2198                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2199                     break;
2200                 case Config.ADMIN_UP:
2201                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2202                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2203                     break;
2204                 case Config.ADMIN_UNDEF:
2205                     break;
2206                 default:
2207                 }
2208             }
2209             break;
2210         case REMOVED:
2211             // This is the case where a switch port is removed from the SDN agent space
2212             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2213             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2214             break;
2215         default:
2216
2217         }
2218
2219         if (updateStaticFlowCluster) {
2220             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2221         }
2222     }
2223
2224     /*
2225      * It goes through the static flows configuration, it identifies the ones
2226      * which have the specified node connector as input or output port and
2227      * install them on the network node if they are marked to be installed in
2228      * hardware and their status shows they were not installed yet
2229      */
2230     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2231         boolean updated = false;
2232         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2233         for (FlowConfig flowConfig : flowConfigForNode) {
2234             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2235                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(StatusCode.SUCCESS.toString())) {
2236                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2237                     if (!status.isSuccess()) {
2238                         flowConfig.setStatus(status.getDescription());
2239                     } else {
2240                         flowConfig.setStatus(StatusCode.SUCCESS.toString());
2241                     }
2242                     updated = true;
2243                 }
2244             }
2245         }
2246         return updated;
2247     }
2248
2249     /*
2250      * Remove from the network node all the flows which have the specified node
2251      * connector as input or output port. If any of the flow entry is a static
2252      * flow, it updates the correspondent configuration.
2253      */
2254     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2255         boolean updated = false;
2256         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2257         if (nodeFlowEntries == null) {
2258             return updated;
2259         }
2260         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2261             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2262                 Status status = this.removeEntryInternal(fei, true);
2263                 if (!status.isSuccess()) {
2264                     continue;
2265                 }
2266                 /*
2267                  * If the flow entry is a static flow, then update its
2268                  * configuration
2269                  */
2270                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2271                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2272                     if (flowConfig != null) {
2273                         flowConfig.setStatus(PORT_REMOVED);
2274                         updated = true;
2275                     }
2276                 }
2277             }
2278         }
2279         return updated;
2280     }
2281
2282     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2283         FlowConfig derivedFlow = new FlowConfig(original);
2284         derivedFlow.setDynamic(true);
2285         derivedFlow.setPortGroup(null);
2286         derivedFlow.setName(original.getName() + "_" + configName + "_" + port);
2287         derivedFlow.setIngressPort(port + "");
2288         return derivedFlow;
2289     }
2290
2291     private void addPortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2292         for (FlowConfig staticFlow : staticFlows.values()) {
2293             if (staticFlow.getPortGroup() == null) {
2294                 continue;
2295             }
2296             if ((staticFlow.getNode().equals(node)) && (staticFlow.getPortGroup().equals(config.getName()))) {
2297                 for (Short port : data.getPorts()) {
2298                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2299                     addStaticFlowInternal(derivedFlow, false);
2300                 }
2301             }
2302         }
2303     }
2304
2305     private void removePortGroupFlows(PortGroupConfig config, Node node, PortGroup data) {
2306         for (FlowConfig staticFlow : staticFlows.values()) {
2307             if (staticFlow.getPortGroup() == null) {
2308                 continue;
2309             }
2310             if (staticFlow.getNode().equals(node) && staticFlow.getPortGroup().equals(config.getName())) {
2311                 for (Short port : data.getPorts()) {
2312                     FlowConfig derivedFlow = getDerivedFlowConfig(staticFlow, config.getName(), port);
2313                     removeStaticFlow(derivedFlow);
2314                 }
2315             }
2316         }
2317     }
2318
2319     @Override
2320     public void portGroupChanged(PortGroupConfig config, Map<Node, PortGroup> data, boolean add) {
2321         log.info("PortGroup Changed for: {} Data: {}", config, portGroupData);
2322         Map<Node, PortGroup> existingData = portGroupData.get(config);
2323         if (existingData != null) {
2324             for (Map.Entry<Node, PortGroup> entry : data.entrySet()) {
2325                 PortGroup existingPortGroup = existingData.get(entry.getKey());
2326                 if (existingPortGroup == null) {
2327                     if (add) {
2328                         existingData.put(entry.getKey(), entry.getValue());
2329                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2330                     }
2331                 } else {
2332                     if (add) {
2333                         existingPortGroup.getPorts().addAll(entry.getValue().getPorts());
2334                         addPortGroupFlows(config, entry.getKey(), entry.getValue());
2335                     } else {
2336                         existingPortGroup.getPorts().removeAll(entry.getValue().getPorts());
2337                         removePortGroupFlows(config, entry.getKey(), entry.getValue());
2338                     }
2339                 }
2340             }
2341         } else {
2342             if (add) {
2343                 portGroupData.put(config, data);
2344                 for (Node swid : data.keySet()) {
2345                     addPortGroupFlows(config, swid, data.get(swid));
2346                 }
2347             }
2348         }
2349     }
2350
2351     @Override
2352     public boolean addPortGroupConfig(String name, String regex, boolean restore) {
2353         PortGroupConfig config = portGroupConfigs.get(name);
2354         if (config != null) {
2355             return false;
2356         }
2357
2358         if ((portGroupProvider == null) && !restore) {
2359             return false;
2360         }
2361         if ((portGroupProvider != null) && (!portGroupProvider.isMatchCriteriaSupported(regex))) {
2362             return false;
2363         }
2364
2365         config = new PortGroupConfig(name, regex);
2366         portGroupConfigs.put(name, config);
2367         if (portGroupProvider != null) {
2368             portGroupProvider.createPortGroupConfig(config);
2369         }
2370         return true;
2371     }
2372
2373     @Override
2374     public boolean delPortGroupConfig(String name) {
2375         PortGroupConfig config = portGroupConfigs.get(name);
2376         if (config == null) {
2377             return false;
2378         }
2379
2380         if (portGroupProvider != null) {
2381             portGroupProvider.deletePortGroupConfig(config);
2382         }
2383         portGroupConfigs.remove(name);
2384         return true;
2385     }
2386
2387     @Override
2388     public Map<String, PortGroupConfig> getPortGroupConfigs() {
2389         return portGroupConfigs;
2390     }
2391
2392     public boolean isPortGroupSupported() {
2393         if (portGroupProvider == null) {
2394             return false;
2395         }
2396         return true;
2397     }
2398
2399     public void setIContainer(IContainer s) {
2400         this.container = s;
2401     }
2402
2403     public void unsetIContainer(IContainer s) {
2404         if (this.container == s) {
2405             this.container = null;
2406         }
2407     }
2408
2409     @Override
2410     public PortGroupProvider getPortGroupProvider() {
2411         return portGroupProvider;
2412     }
2413
2414     public void unsetPortGroupProvider(PortGroupProvider portGroupProvider) {
2415         this.portGroupProvider = null;
2416     }
2417
2418     public void setPortGroupProvider(PortGroupProvider portGroupProvider) {
2419         this.portGroupProvider = portGroupProvider;
2420         portGroupProvider.registerPortGroupChange(this);
2421         for (PortGroupConfig config : portGroupConfigs.values()) {
2422             portGroupProvider.createPortGroupConfig(config);
2423         }
2424     }
2425
2426     public void setFrmAware(IForwardingRulesManagerAware obj) {
2427         this.frmAware.add(obj);
2428     }
2429
2430     public void unsetFrmAware(IForwardingRulesManagerAware obj) {
2431         this.frmAware.remove(obj);
2432     }
2433
2434     void setClusterContainerService(IClusterContainerServices s) {
2435         log.debug("Cluster Service set");
2436         this.clusterContainerService = s;
2437     }
2438
2439     void unsetClusterContainerService(IClusterContainerServices s) {
2440         if (this.clusterContainerService == s) {
2441             log.debug("Cluster Service removed!");
2442             this.clusterContainerService = null;
2443         }
2444     }
2445
2446     private String getContainerName() {
2447         if (container == null) {
2448             return GlobalConstants.DEFAULT.toString();
2449         }
2450         return container.getName();
2451     }
2452
2453     /**
2454      * Function called by the dependency manager when all the required
2455      * dependencies are satisfied
2456      *
2457      */
2458     void init() {
2459         frmFileName = GlobalConstants.STARTUPHOME.toString() + "frm_staticflows_" + this.getContainerName() + ".conf";
2460         portGroupFileName = GlobalConstants.STARTUPHOME.toString() + "portgroup_" + this.getContainerName() + ".conf";
2461
2462         inContainerMode = false;
2463
2464         if (portGroupProvider != null) {
2465             portGroupProvider.registerPortGroupChange(this);
2466         }
2467
2468         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
2469         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
2470
2471         cacheStartup();
2472
2473         registerWithOSGIConsole();
2474
2475         /*
2476          * If we are not the first cluster node to come up, do not initialize
2477          * the static flow entries ordinal
2478          */
2479         if (staticFlowsOrdinal.size() == 0) {
2480             staticFlowsOrdinal.put(0, Integer.valueOf(0));
2481         }
2482
2483         pendingEvents = new LinkedBlockingQueue<FRMEvent>();
2484
2485         // Initialize the event handler thread
2486         frmEventHandler = new Thread(new Runnable() {
2487             @Override
2488             public void run() {
2489                 while (!stopping) {
2490                     try {
2491                         final FRMEvent event = pendingEvents.take();
2492                         if (event == null) {
2493                             log.warn("Dequeued null event");
2494                             continue;
2495                         }
2496                         log.trace("Dequeued {} event", event.getClass().getSimpleName());
2497                         if (event instanceof NodeUpdateEvent) {
2498                             NodeUpdateEvent update = (NodeUpdateEvent) event;
2499                             Node node = update.getNode();
2500                             switch (update.getUpdateType()) {
2501                             case ADDED:
2502                                 addStaticFlowsToSwitch(node);
2503                                 break;
2504                             case REMOVED:
2505                                 cleanDatabaseForNode(node);
2506                                 updateStaticFlowConfigsOnNodeDown(node);
2507                                 break;
2508                             default:
2509                             }
2510                         } else if (event instanceof ErrorReportedEvent) {
2511                             ErrorReportedEvent errEvent = (ErrorReportedEvent) event;
2512                             processErrorEvent(errEvent);
2513                         } else if (event instanceof WorkOrderEvent) {
2514                             /*
2515                              * Take care of handling the remote Work request
2516                              */
2517                             Runnable r = new Runnable() {
2518                                 @Override
2519                                 public void run() {
2520                                     WorkOrderEvent work = (WorkOrderEvent) event;
2521                                     FlowEntryDistributionOrder fe = work.getFe();
2522                                     if (fe != null) {
2523                                         logsync.trace("Executing the workOrder {}", fe);
2524                                         Status gotStatus = null;
2525                                         FlowEntryInstall feiCurrent = fe.getEntry();
2526                                         FlowEntryInstall feiNew = workOrder.get(fe);
2527                                         switch (fe.getUpType()) {
2528                                         case ADDED: