c58410c668ca3a7d72ee80d020c8a251dd761407
[controller.git] / opendaylight / forwardingrulesmanager / implementation / src / main / java / org / opendaylight / controller / forwardingrulesmanager / internal / ForwardingRulesManager.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.forwardingrulesmanager.internal;
10
11 import java.io.FileNotFoundException;
12 import java.io.IOException;
13 import java.io.ObjectInputStream;
14 import java.net.InetAddress;
15 import java.net.UnknownHostException;
16 import java.util.ArrayList;
17 import java.util.Collections;
18 import java.util.EnumSet;
19 import java.util.HashSet;
20 import java.util.List;
21 import java.util.Map;
22 import java.util.Map.Entry;
23 import java.util.Set;
24 import java.util.concurrent.BlockingQueue;
25 import java.util.concurrent.Callable;
26 import java.util.concurrent.ConcurrentHashMap;
27 import java.util.concurrent.ConcurrentMap;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorService;
30 import java.util.concurrent.Executors;
31 import java.util.concurrent.Future;
32 import java.util.concurrent.LinkedBlockingQueue;
33
34 import org.eclipse.osgi.framework.console.CommandInterpreter;
35 import org.eclipse.osgi.framework.console.CommandProvider;
36 import org.opendaylight.controller.clustering.services.CacheConfigException;
37 import org.opendaylight.controller.clustering.services.CacheExistException;
38 import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
39 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
40 import org.opendaylight.controller.clustering.services.IClusterServices;
41 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
42 import org.opendaylight.controller.connectionmanager.ConnectionLocality;
43 import org.opendaylight.controller.connectionmanager.IConnectionManager;
44 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
45 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
46 import org.opendaylight.controller.forwardingrulesmanager.FlowEntryInstall;
47 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
48 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManagerAware;
49 import org.opendaylight.controller.forwardingrulesmanager.PortGroup;
50 import org.opendaylight.controller.forwardingrulesmanager.PortGroupChangeListener;
51 import org.opendaylight.controller.forwardingrulesmanager.PortGroupConfig;
52 import org.opendaylight.controller.forwardingrulesmanager.PortGroupProvider;
53 import org.opendaylight.controller.forwardingrulesmanager.implementation.data.FlowEntryDistributionOrder;
54 import org.opendaylight.controller.sal.action.Action;
55 import org.opendaylight.controller.sal.action.ActionType;
56 import org.opendaylight.controller.sal.action.Controller;
57 import org.opendaylight.controller.sal.action.Flood;
58 import org.opendaylight.controller.sal.action.Output;
59 import org.opendaylight.controller.sal.action.PopVlan;
60 import org.opendaylight.controller.sal.core.Config;
61 import org.opendaylight.controller.sal.core.ContainerFlow;
62 import org.opendaylight.controller.sal.core.IContainer;
63 import org.opendaylight.controller.sal.core.IContainerListener;
64 import org.opendaylight.controller.sal.core.Node;
65 import org.opendaylight.controller.sal.core.NodeConnector;
66 import org.opendaylight.controller.sal.core.Property;
67 import org.opendaylight.controller.sal.core.UpdateType;
68 import org.opendaylight.controller.sal.flowprogrammer.Flow;
69 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerListener;
70 import org.opendaylight.controller.sal.flowprogrammer.IFlowProgrammerService;
71 import org.opendaylight.controller.sal.match.Match;
72 import org.opendaylight.controller.sal.match.MatchType;
73 import org.opendaylight.controller.sal.utils.EtherTypes;
74 import org.opendaylight.controller.sal.utils.GlobalConstants;
75 import org.opendaylight.controller.sal.utils.HexEncode;
76 import org.opendaylight.controller.sal.utils.IObjectReader;
77 import org.opendaylight.controller.sal.utils.IPProtocols;
78 import org.opendaylight.controller.sal.utils.NodeConnectorCreator;
79 import org.opendaylight.controller.sal.utils.NodeCreator;
80 import org.opendaylight.controller.sal.utils.ObjectReader;
81 import org.opendaylight.controller.sal.utils.ObjectWriter;
82 import org.opendaylight.controller.sal.utils.Status;
83 import org.opendaylight.controller.sal.utils.StatusCode;
84 import org.opendaylight.controller.switchmanager.IInventoryListener;
85 import org.opendaylight.controller.switchmanager.ISwitchManager;
86 import org.opendaylight.controller.switchmanager.ISwitchManagerAware;
87 import org.opendaylight.controller.switchmanager.Subnet;
88 import org.osgi.framework.BundleContext;
89 import org.osgi.framework.FrameworkUtil;
90 import org.slf4j.Logger;
91 import org.slf4j.LoggerFactory;
92
93 /**
94  * Class that manages forwarding rule installation and removal per container of
95  * the network. It also maintains the central repository of all the forwarding
96  * rules installed on the network nodes.
97  */
98 public class ForwardingRulesManager implements
99         IForwardingRulesManager,
100         PortGroupChangeListener,
101         IContainerListener,
102         ISwitchManagerAware,
103         IConfigurationContainerAware,
104         IInventoryListener,
105         IObjectReader,
106         ICacheUpdateAware<Object,Object>,
107         CommandProvider,
108         IFlowProgrammerListener {
109     private static final String NODEDOWN = "Node is Down";
110     private static final String SUCCESS = StatusCode.SUCCESS.toString();
111     private static final Logger log = LoggerFactory.getLogger(ForwardingRulesManager.class);
112     private static final String PORTREMOVED = "Port removed";
113     private static final Logger logsync = LoggerFactory.getLogger("FRMsync");
114     private String frmFileName;
115     private String portGroupFileName;
116     private ConcurrentMap<Integer, FlowConfig> staticFlows;
117     private ConcurrentMap<Integer, Integer> staticFlowsOrdinal;
118     private ConcurrentMap<String, PortGroupConfig> portGroupConfigs;
119     private ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>> portGroupData;
120     private ConcurrentMap<String, Object> TSPolicies;
121     private boolean inContainerMode; // being used by global instance only
122     private boolean stopping;
123
124     /*
125      * Flow database. It's the software view of what was requested to install
126      * and what is installed on the switch. It is indexed by the entry itself.
127      * The entry's hashcode resumes the network node index, the flow's priority
128      * and the flow's match. The value element is a class which contains the
129      * flow entry pushed by the applications modules and the respective
130      * container flow merged version. In absence of container flows, the two
131      * flow entries are the same.
132      */
133     private ConcurrentMap<FlowEntry, FlowEntry> originalSwView;
134     private ConcurrentMap<FlowEntryInstall, FlowEntryInstall> installedSwView;
135     /*
136      * Per node and per group indexing
137      */
138     private ConcurrentMap<Node, List<FlowEntryInstall>> nodeFlows;
139     private ConcurrentMap<String, List<FlowEntryInstall>> groupFlows;
140
141     /*
142      * Inactive flow list. This is for the global instance of FRM It will
143      * contain all the flow entries which were installed on the global container
144      * when the first container is created.
145      */
146     private ConcurrentMap<FlowEntry, FlowEntry> inactiveFlows;
147
148     private IContainer container;
149     private Set<IForwardingRulesManagerAware> frmAware =
150         Collections.synchronizedSet(new HashSet<IForwardingRulesManagerAware>());
151     private PortGroupProvider portGroupProvider;
152     private IFlowProgrammerService programmer;
153     private IClusterContainerServices clusterContainerService = null;
154     private ISwitchManager switchManager;
155     private Thread frmEventHandler;
156     protected BlockingQueue<FRMEvent> pendingEvents;
157
158     // Distributes FRM programming in the cluster
159     private IConnectionManager connectionManager;
160
161     /*
162      * Name clustered caches used to support FRM entry distribution these are by
163      * necessity non-transactional as long as need to be able to synchronize
164      * states also while a transaction is in progress
165      */
166     static final String WORKORDERCACHE = "frm.workOrder";
167     static final String WORKSTATUSCACHE = "frm.workStatus";
168
169     /*
170      * Data structure responsible for distributing the FlowEntryInstall requests
171      * in the cluster. The key value is entry that is being either Installed or
172      * Updated or Delete. The value field is the same of the key value in case
173      * of Installation or Deletion, it's the new entry in case of Modification,
174      * this because the clustering caches don't allow null values.
175      *
176      * The logic behind this data structure is that the controller that initiate
177      * the request will place the order here, someone will pick it and then will
178      * remove from this data structure because is being served.
179      *
180      * TODO: We need to have a way to cleanup this data structure if entries are
181      * not picked by anyone, which is always a case can happen especially on
182      * Node disconnect cases.
183      */
184     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall> workOrder;
185
186     /*
187      * Data structure responsible for retrieving the results of the workOrder
188      * submitted to the cluster.
189      *
190      * The logic behind this data structure is that the controller that has
191      * executed the order will then place the result in workStatus signaling
192      * that there was a success or a failure.
193      *
194      * TODO: The workStatus entries need to have a lifetime associated in case
195      * of requestor controller leaving the cluster.
196      */
197     private ConcurrentMap<FlowEntryDistributionOrder, Status> workStatus;
198
199     /*
200      * Local Map used to hold the Future which a caller can use to monitor for
201      * completion
202      */
203     private ConcurrentMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask> workMonitor =
204             new ConcurrentHashMap<FlowEntryDistributionOrder, FlowEntryDistributionOrderFutureTask>();
205
206     /**
207      * @param e
208      *            Entry being installed/updated/removed
209      * @param u
210      *            New entry will be placed after the update operation. Valid
211      *            only for UpdateType.CHANGED, null for all the other cases
212      * @param t
213      *            Type of update
214      * @return a Future object for monitoring the progress of the result, or
215      *         null in case the processing should take place locally
216      */
217     private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
218             UpdateType t) {
219         // A null entry it's an unexpected condition, anyway it's safe to keep
220         // the handling local
221         if (e == null) {
222             return null;
223         }
224
225         Node n = e.getNode();
226         if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
227             // Create the work order and distribute it
228             FlowEntryDistributionOrder fe =
229                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
230             // First create the monitor job
231             FlowEntryDistributionOrderFutureTask ret = new FlowEntryDistributionOrderFutureTask(fe);
232             logsync.trace("Node {} not local so sending fe {}", n, fe);
233             workMonitor.put(fe, ret);
234             if (t.equals(UpdateType.CHANGED)) {
235                 // Then distribute the work
236                 workOrder.put(fe, u);
237             } else {
238                 // Then distribute the work
239                 workOrder.put(fe, e);
240             }
241             logsync.trace("WorkOrder requested");
242             // Now create an Handle to monitor the execution of the operation
243             return ret;
244         }
245
246         logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
247         return null;
248     }
249
250     /**
251      * Adds a flow entry onto the network node It runs various validity checks
252      * and derive the final container flows merged entries that will be
253      * attempted to be installed
254      *
255      * @param flowEntry
256      *            the original flow entry application requested to add
257      * @param async
258      *            the flag indicating if this is a asynchronous request
259      * @return the status of this request. In case of asynchronous call, it will
260      *         contain the unique id assigned to this request
261      */
262     private Status addEntry(FlowEntry flowEntry, boolean async) {
263
264         // Sanity Check
265         if (flowEntry == null || flowEntry.getNode() == null) {
266             String msg = "Invalid FlowEntry";
267             String logMsg = msg + ": {}";
268             log.warn(logMsg, flowEntry);
269             return new Status(StatusCode.NOTACCEPTABLE, msg);
270         }
271
272         /*
273          * Derive the container flow merged entries to install In presence of N
274          * container flows, we may end up with N different entries to install...
275          */
276         List<FlowEntryInstall> toInstallList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
277
278         // Container Flow conflict Check
279         if (toInstallList.isEmpty()) {
280             String msg = "Flow Entry conflicts with all Container Flows";
281             String logMsg = msg + ": {}";
282             log.warn(logMsg, flowEntry);
283             return new Status(StatusCode.CONFLICT, msg);
284         }
285
286         // Derive the list of entries good to be installed
287         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
288         for (FlowEntryInstall entry : toInstallList) {
289             // Conflict Check: Verify new entry would not overwrite existing
290             // ones
291             if (this.installedSwView.containsKey(entry)) {
292                 log.warn("Operation Rejected: A flow with same match and priority exists on the target node");
293                 log.trace("Aborting to install {}", entry);
294                 continue;
295             }
296             toInstallSafe.add(entry);
297         }
298
299         // Declare failure if all the container flow merged entries clash with
300         // existing entries
301         if (toInstallSafe.size() == 0) {
302             String msg = "A flow with same match and priority exists on the target node";
303             String logMsg = msg + ": {}";
304             log.warn(logMsg, flowEntry);
305             return new Status(StatusCode.CONFLICT, msg);
306         }
307
308         // Try to install an entry at the time
309         Status error = new Status(null, null);
310         Status succeded = null;
311         boolean oneSucceded = false;
312         for (FlowEntryInstall installEntry : toInstallSafe) {
313
314             // Install and update database
315             Status ret = addEntriesInternal(installEntry, async);
316
317             if (ret.isSuccess()) {
318                 oneSucceded = true;
319                 /*
320                  * The first successful status response will be returned For the
321                  * asynchronous call, we can discard the container flow
322                  * complication for now and assume we will always deal with one
323                  * flow only per request
324                  */
325                 succeded = ret;
326             } else {
327                 error = ret;
328                 log.warn("Failed to install the entry: {}. The failure is: {}", installEntry, ret.getDescription());
329             }
330         }
331
332         return (oneSucceded) ? succeded : error;
333     }
334
335     /**
336      * Given a flow entry and the list of container flows, it returns the list
337      * of container flow merged flow entries good to be installed on this
338      * container. If the list of container flows is null or empty, the install
339      * entry list will contain only one entry, the original flow entry. If the
340      * flow entry is congruent with all the N container flows, then the output
341      * install entry list will contain N entries. If the output list is empty,
342      * it means the passed flow entry conflicts with all the container flows.
343      *
344      * @param cFlowList
345      *            The list of container flows
346      * @return the list of container flow merged entries good to be installed on
347      *         this container
348      */
349     private List<FlowEntryInstall> deriveInstallEntries(FlowEntry request, List<ContainerFlow> cFlowList) {
350         List<FlowEntryInstall> toInstallList = new ArrayList<FlowEntryInstall>(1);
351
352         if (container.getContainerFlows() == null || container.getContainerFlows().isEmpty()) {
353             // No container flows => entry good to be installed unchanged
354             toInstallList.add(new FlowEntryInstall(request.clone(), null));
355         } else {
356             // Create the list of entries to be installed. If the flow entry is
357             // not congruent with any container flow, no install entries will be
358             // created
359             for (ContainerFlow cFlow : container.getContainerFlows()) {
360                 if (cFlow.allowsFlow(request.getFlow())) {
361                     toInstallList.add(new FlowEntryInstall(request.clone(), cFlow));
362                 }
363             }
364         }
365         return toInstallList;
366     }
367
368     /**
369      * Modify a flow entry with a new one It runs various validity check and
370      * derive the final container flows merged flow entries to work with
371      *
372      * @param currentFlowEntry
373      * @param newFlowEntry
374      * @param async
375      *            the flag indicating if this is a asynchronous request
376      * @return the status of this request. In case of asynchronous call, it will
377      *         contain the unique id assigned to this request
378      */
379     private Status modifyEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry, boolean async) {
380         Status retExt;
381
382         // Sanity checks
383         if (currentFlowEntry == null || currentFlowEntry.getNode() == null || newFlowEntry == null
384                 || newFlowEntry.getNode() == null) {
385             String msg = "Modify: Invalid FlowEntry";
386             String logMsg = msg + ": {} or {}";
387             log.warn(logMsg, currentFlowEntry, newFlowEntry);
388             return new Status(StatusCode.NOTACCEPTABLE, msg);
389         }
390         if (!currentFlowEntry.getNode().equals(newFlowEntry.getNode())
391                 || !currentFlowEntry.getFlowName().equals(newFlowEntry.getFlowName())) {
392             String msg = "Modify: Incompatible Flow Entries";
393             String logMsg = msg + ": {} and {}";
394             log.warn(logMsg, currentFlowEntry, newFlowEntry);
395             return new Status(StatusCode.NOTACCEPTABLE, msg);
396         }
397
398         // Equality Check
399         if (currentFlowEntry.getFlow().equals(newFlowEntry.getFlow())) {
400             String msg = "Modify skipped as flows are the same";
401             String logMsg = msg + ": {} and {}";
402             log.debug(logMsg, currentFlowEntry, newFlowEntry);
403             return new Status(StatusCode.SUCCESS, msg);
404         }
405
406         /*
407          * Conflict Check: Verify the new entry would not conflict with an
408          * existing one. This is a loose check on the previous original flow
409          * entry requests. No check on the container flow merged flow entries
410          * (if any) yet
411          */
412         FlowEntry sameMatchOriginalEntry = originalSwView.get(newFlowEntry);
413         if (sameMatchOriginalEntry != null && !sameMatchOriginalEntry.equals(currentFlowEntry)) {
414             String msg = "Operation Rejected: Another flow with same match and priority exists on the target node";
415             String logMsg = msg + ": {}";
416             log.warn(logMsg, currentFlowEntry);
417             return new Status(StatusCode.CONFLICT, msg);
418         }
419
420         // Derive the installed and toInstall entries
421         List<FlowEntryInstall> installedList = deriveInstallEntries(currentFlowEntry.clone(),
422                 container.getContainerFlows());
423         List<FlowEntryInstall> toInstallList = deriveInstallEntries(newFlowEntry.clone(), container.getContainerFlows());
424
425         if (toInstallList.isEmpty()) {
426             String msg = "Modify Operation Rejected: The new entry conflicts with all the container flows";
427             String logMsg = msg + ": {}";
428             log.warn(logMsg, newFlowEntry);
429             log.warn(msg);
430             return new Status(StatusCode.CONFLICT, msg);
431         }
432
433         /*
434          * If the two list sizes differ, it means the new flow entry does not
435          * satisfy the same number of container flows the current entry does.
436          * This is only possible when the new entry and current entry have
437          * different match. In this scenario the modification would ultimately
438          * be handled as a remove and add operations in the protocol plugin.
439          *
440          * Also, if any of the new flow entries would clash with an existing
441          * one, we cannot proceed with the modify operation, because it would
442          * fail for some entries and leave stale entries on the network node.
443          * Modify path can be taken only if it can be performed completely, for
444          * all entries.
445          *
446          * So, for the above two cases, to simplify, let's decouple the modify
447          * in: 1) remove current entries 2) install new entries
448          */
449         Status succeeded = null;
450         boolean decouple = false;
451         if (installedList.size() != toInstallList.size()) {
452             log.info("Modify: New flow entry does not satisfy the same "
453                     + "number of container flows as the original entry does");
454             decouple = true;
455         }
456         List<FlowEntryInstall> toInstallSafe = new ArrayList<FlowEntryInstall>();
457         for (FlowEntryInstall installEntry : toInstallList) {
458             /*
459              * Conflict Check: Verify the new entry would not overwrite another
460              * existing one
461              */
462             FlowEntryInstall sameMatchEntry = installedSwView.get(installEntry);
463             if (sameMatchEntry != null && !sameMatchEntry.getOriginal().equals(currentFlowEntry)) {
464                 log.info("Modify: new container flow merged flow entry clashes with existing flow");
465                 decouple = true;
466             } else {
467                 toInstallSafe.add(installEntry);
468             }
469         }
470
471         if (decouple) {
472             // Remove current entries
473             for (FlowEntryInstall currEntry : installedList) {
474                 this.removeEntryInternal(currEntry, async);
475             }
476             // Install new entries
477             for (FlowEntryInstall newEntry : toInstallSafe) {
478                 succeeded = this.addEntriesInternal(newEntry, async);
479             }
480         } else {
481             /*
482              * The two list have the same size and the entries to install do not
483              * clash with any existing flow on the network node. We assume here
484              * (and might be wrong) that the same container flows that were
485              * satisfied by the current entries are the same that are satisfied
486              * by the new entries. Let's take the risk for now.
487              *
488              * Note: modification has to be complete. If any entry modification
489              * fails, we need to stop, restore the already modified entries, and
490              * declare failure.
491              */
492             Status retModify = null;
493             int i = 0;
494             int size = toInstallList.size();
495             while (i < size) {
496                 // Modify and update database
497                 retModify = modifyEntryInternal(installedList.get(i), toInstallList.get(i), async);
498                 if (retModify.isSuccess()) {
499                     i++;
500                 } else {
501                     break;
502                 }
503             }
504             // Check if uncompleted modify
505             if (i < size) {
506                 log.warn("Unable to perform a complete modify for all  the container flows merged entries");
507                 // Restore original entries
508                 int j = 0;
509                 while (j < i) {
510                     log.info("Attempting to restore initial entries");
511                     retExt = modifyEntryInternal(toInstallList.get(i), installedList.get(i), async);
512                     if (retExt.isSuccess()) {
513                         j++;
514                     } else {
515                         break;
516                     }
517                 }
518                 // Fatal error, recovery failed
519                 if (j < i) {
520                     String msg = "Flow recovery failed ! Unrecoverable Error";
521                     log.error(msg);
522                     return new Status(StatusCode.INTERNALERROR, msg);
523                 }
524             }
525             succeeded = retModify;
526         }
527         /*
528          * The first successful status response will be returned. For the
529          * asynchronous call, we can discard the container flow complication for
530          * now and assume we will always deal with one flow only per request
531          */
532         return succeeded;
533     }
534
535     /**
536      * This is the function that modifies the final container flows merged
537      * entries on the network node and update the database. It expects that all
538      * the validity checks are passed
539      *
540      * @param currentEntries
541      * @param newEntries
542      * @param async
543      *            the flag indicating if this is a asynchronous request
544      * @return the status of this request. In case of asynchronous call, it will
545      *         contain the unique id assigned to this request
546      */
547     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
548         FlowEntryDistributionOrderFutureTask futureStatus =
549                 distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
550         if (futureStatus != null) {
551             Status retStatus = new Status(StatusCode.UNDEFINED);
552             try {
553                 retStatus = futureStatus.get();
554                 if (retStatus.getCode()
555                         .equals(StatusCode.TIMEOUT)) {
556                     // A timeout happened, lets cleanup the workMonitor
557                     workMonitor.remove(futureStatus.getOrder());
558                 }
559             } catch (InterruptedException e) {
560                 log.error("", e);
561             } catch (ExecutionException e) {
562                 log.error("", e);
563             }
564             return retStatus;
565         } else {
566             // Modify the flow on the network node
567             Status status = async ? programmer.modifyFlowAsync(currentEntries.getNode(), currentEntries.getInstall()
568                     .getFlow(), newEntries.getInstall()
569                     .getFlow()) : programmer.modifyFlow(currentEntries.getNode(), currentEntries.getInstall()
570                     .getFlow(), newEntries.getInstall()
571                     .getFlow());
572
573             if (!status.isSuccess()) {
574                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", newEntries.getInstall(),
575                         status.getDescription());
576                 return status;
577             }
578
579             log.trace("Modified {} => {}", currentEntries.getInstall(), newEntries.getInstall());
580
581             // Update DB
582             newEntries.setRequestId(status.getRequestId());
583             updateLocalDatabase(currentEntries, false);
584             updateLocalDatabase(newEntries, true);
585
586             return status;
587         }
588     }
589
590     /**
591      * Remove a flow entry. If the entry is not present in the software view
592      * (entry or node not present), it return successfully
593      *
594      * @param flowEntry
595      *            the flow entry to remove
596      * @param async
597      *            the flag indicating if this is a asynchronous request
598      * @return the status of this request. In case of asynchronous call, it will
599      *         contain the unique id assigned to this request
600      */
601     private Status removeEntry(FlowEntry flowEntry, boolean async) {
602         Status error = new Status(null, null);
603
604         // Sanity Check
605         if (flowEntry == null || flowEntry.getNode() == null) {
606             String msg = "Invalid FlowEntry";
607             String logMsg = msg + ": {}";
608             log.warn(logMsg, flowEntry);
609             return new Status(StatusCode.NOTACCEPTABLE, msg);
610         }
611
612         // Derive the container flows merged installed entries
613         List<FlowEntryInstall> installedList = deriveInstallEntries(flowEntry.clone(), container.getContainerFlows());
614
615         Status succeeded = null;
616         boolean atLeastOneRemoved = false;
617         for (FlowEntryInstall entry : installedList) {
618             if (!installedSwView.containsKey(entry)) {
619                 String logMsg = "Removal skipped (not present in software view) for flow entry: {}";
620                 log.debug(logMsg, flowEntry);
621                 if (installedList.size() == 1) {
622                     // If we had only one entry to remove, we are done
623                     return new Status(StatusCode.SUCCESS);
624                 } else {
625                     continue;
626                 }
627             }
628
629             // Remove and update DB
630             Status ret = removeEntryInternal(entry, async);
631
632             if (!ret.isSuccess()) {
633                 error = ret;
634                 log.warn("Failed to remove the entry: {}. The failure is: {}", entry.getInstall(), ret.getDescription());
635                 if (installedList.size() == 1) {
636                     // If we had only one entry to remove, this is fatal failure
637                     return error;
638                 }
639             } else {
640                 succeeded = ret;
641                 atLeastOneRemoved = true;
642             }
643         }
644
645         /*
646          * No worries if full removal failed. Consistency checker will take care
647          * of removing the stale entries later, or adjusting the software
648          * database if not in sync with hardware
649          */
650         return (atLeastOneRemoved) ? succeeded : error;
651     }
652
653     /**
654      * This is the function that removes the final container flows merged entry
655      * from the network node and update the database. It expects that all the
656      * validity checks are passed
657      *
658      * @param entry
659      *            the flow entry to remove
660      * @param async
661      *            the flag indicating if this is a asynchronous request
662      * @return the status of this request. In case of asynchronous call, it will
663      *         contain the unique id assigned to this request
664      */
665     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
666         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
667         if (futureStatus != null) {
668             Status retStatus = new Status(StatusCode.UNDEFINED);
669             try {
670                 retStatus = futureStatus.get();
671                 if (retStatus.getCode()
672                         .equals(StatusCode.TIMEOUT)) {
673                     // A timeout happened, lets cleanup the workMonitor
674                     workMonitor.remove(futureStatus.getOrder());
675                 }
676             } catch (InterruptedException e) {
677                 log.error("", e);
678             } catch (ExecutionException e) {
679                 log.error("", e);
680             }
681             return retStatus;
682         } else {
683             // Mark the entry to be deleted (for CC just in case we fail)
684             entry.toBeDeleted();
685
686             // Remove from node
687             Status status = async ? programmer.removeFlowAsync(entry.getNode(), entry.getInstall()
688                     .getFlow()) : programmer.removeFlow(entry.getNode(), entry.getInstall()
689                     .getFlow());
690
691             if (!status.isSuccess()) {
692                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
693                         status.getDescription());
694                 return status;
695             }
696             log.trace("Removed  {}", entry.getInstall());
697
698             // Update DB
699             updateLocalDatabase(entry, false);
700
701             return status;
702         }
703     }
704
705     /**
706      * This is the function that installs the final container flow merged entry
707      * on the network node and updates the database. It expects that all the
708      * validity and conflict checks are passed. That means it does not check
709      * whether this flow would conflict or overwrite an existing one.
710      *
711      * @param entry
712      *            the flow entry to install
713      * @param async
714      *            the flag indicating if this is a asynchronous request
715      * @return the status of this request. In case of asynchronous call, it will
716      *         contain the unique id assigned to this request
717      */
718     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
719         FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
720         if (futureStatus != null) {
721             Status retStatus = new Status(StatusCode.UNDEFINED);
722             try {
723                 retStatus = futureStatus.get();
724                 if (retStatus.getCode()
725                         .equals(StatusCode.TIMEOUT)) {
726                     // A timeout happened, lets cleanup the workMonitor
727                     workMonitor.remove(futureStatus.getOrder());
728                 }
729             } catch (InterruptedException e) {
730                 log.error("", e);
731             } catch (ExecutionException e) {
732                 log.error("", e);
733             }
734             return retStatus;
735         } else {
736             // Install the flow on the network node
737             Status status = async ? programmer.addFlowAsync(entry.getNode(), entry.getInstall()
738                     .getFlow()) : programmer.addFlow(entry.getNode(), entry.getInstall()
739                     .getFlow());
740
741             if (!status.isSuccess()) {
742                 log.warn("SDN Plugin failed to program the flow: {}. The failure is: {}", entry.getInstall(),
743                         status.getDescription());
744                 return status;
745             }
746
747             log.trace("Added    {}", entry.getInstall());
748
749             // Update DB
750             entry.setRequestId(status.getRequestId());
751             updateLocalDatabase(entry, true);
752
753             return status;
754         }
755     }
756
757     /**
758      * Returns true if the flow conflicts with all the container's flows. This
759      * means that if the function returns true, the passed flow entry is
760      * congruent with at least one container flow, hence it is good to be
761      * installed on this container.
762      *
763      * @param flowEntry
764      * @return true if flow conflicts with all the container flows, false
765      *         otherwise
766      */
767     private boolean entryConflictsWithContainerFlows(FlowEntry flowEntry) {
768         List<ContainerFlow> cFlowList = container.getContainerFlows();
769
770         // Validity check and avoid unnecessary computation
771         // Also takes care of default container where no container flows are
772         // present
773         if (cFlowList == null || cFlowList.isEmpty()) {
774             return false;
775         }
776
777         for (ContainerFlow cFlow : cFlowList) {
778             if (cFlow.allowsFlow(flowEntry.getFlow())) {
779                 // Entry is allowed by at least one container flow: good to go
780                 return false;
781             }
782         }
783         return true;
784     }
785
786     private void updateLocalDatabase(FlowEntryInstall entry, boolean add) {
787         // Update the software view
788         updateSwViewes(entry, add);
789
790         // Update node indexed flow database
791         updateNodeFlowsDB(entry, add);
792
793         // Update group indexed flow database
794         updateGroupFlowsDB(entry, add);
795     }
796
797     /*
798      * Update the node mapped flows database
799      */
800     private void updateSwViewes(FlowEntryInstall flowEntries, boolean add) {
801         if (add) {
802             originalSwView.put(flowEntries.getOriginal(), flowEntries.getOriginal());
803             installedSwView.put(flowEntries, flowEntries);
804         } else {
805             originalSwView.remove(flowEntries.getOriginal());
806             installedSwView.remove(flowEntries);
807         }
808     }
809
810     /*
811      * Update the node mapped flows database
812      */
813     private void updateNodeFlowsDB(FlowEntryInstall flowEntries, boolean add) {
814         Node node = flowEntries.getNode();
815
816         List<FlowEntryInstall> nodeIndeces = this.nodeFlows.get(node);
817         if (nodeIndeces == null) {
818             if (!add) {
819                 return;
820             } else {
821                 nodeIndeces = new ArrayList<FlowEntryInstall>();
822             }
823         }
824
825         if (add) {
826             nodeIndeces.add(flowEntries);
827         } else {
828             nodeIndeces.remove(flowEntries);
829         }
830
831         // Update cache across cluster
832         if (nodeIndeces.isEmpty()) {
833             this.nodeFlows.remove(node);
834         } else {
835             this.nodeFlows.put(node, nodeIndeces);
836         }
837     }
838
839     /*
840      * Update the group name mapped flows database
841      */
842     private void updateGroupFlowsDB(FlowEntryInstall flowEntries, boolean add) {
843         String groupName = flowEntries.getGroupName();
844
845         // Flow may not be part of a group
846         if (groupName == null) {
847             return;
848         }
849
850         List<FlowEntryInstall> indices = this.groupFlows.get(groupName);
851         if (indices == null) {
852             if (!add) {
853                 return;
854             } else {
855                 indices = new ArrayList<FlowEntryInstall>();
856             }
857         }
858
859         if (add) {
860             indices.add(flowEntries);
861         } else {
862             indices.remove(flowEntries);
863         }
864
865         // Update cache across cluster
866         if (indices.isEmpty()) {
867             this.groupFlows.remove(groupName);
868         } else {
869             this.groupFlows.put(groupName, indices);
870         }
871     }
872
873     /**
874      * Remove a flow entry that has been added previously First checks if the
875      * entry is effectively present in the local database
876      */
877     @SuppressWarnings("unused")
878     private Status removeEntry(Node node, String flowName) {
879         FlowEntryInstall target = null;
880
881         // Find in database
882         for (FlowEntryInstall entry : installedSwView.values()) {
883             if (entry.equalsByNodeAndName(node, flowName)) {
884                 target = entry;
885                 break;
886             }
887         }
888
889         // If it is not there, stop any further processing
890         if (target == null) {
891             return new Status(StatusCode.SUCCESS, "Entry is not present");
892         }
893
894         // Remove from node
895         Status status = programmer.removeFlow(target.getNode(), target.getInstall().getFlow());
896
897         // Update DB
898         if (status.isSuccess()) {
899             updateLocalDatabase(target, false);
900         } else {
901             // log the error
902             log.warn("SDN Plugin failed to remove the flow: {}. The failure is: {}", target.getInstall(),
903                     status.getDescription());
904         }
905
906         return status;
907     }
908
909     @Override
910     public Status installFlowEntry(FlowEntry flowEntry) {
911         Status status;
912         if (isContainerModeAllowed(flowEntry)) {
913             status = addEntry(flowEntry, false);
914         } else {
915             String msg = "Controller in container mode: Install Refused";
916             String logMsg = msg + ": {}";
917             status = new Status(StatusCode.NOTACCEPTABLE, msg);
918             log.warn(logMsg, flowEntry);
919         }
920         return status;
921     }
922
923     @Override
924     public Status installFlowEntryAsync(FlowEntry flowEntry) {
925         Status status;
926         if (isContainerModeAllowed(flowEntry)) {
927             status = addEntry(flowEntry, true);
928         } else {
929             String msg = "Controller in container mode: Install Refused";
930             status = new Status(StatusCode.NOTACCEPTABLE, msg);
931             log.warn(msg);
932         }
933         return status;
934     }
935
936     @Override
937     public Status uninstallFlowEntry(FlowEntry flowEntry) {
938         Status status;
939         if (isContainerModeAllowed(flowEntry)) {
940             status = removeEntry(flowEntry, false);
941         } else {
942             String msg = "Controller in container mode: Uninstall Refused";
943             String logMsg = msg + ": {}";
944             status = new Status(StatusCode.NOTACCEPTABLE, msg);
945             log.warn(logMsg, flowEntry);
946         }
947         return status;
948     }
949
950     @Override
951     public Status uninstallFlowEntryAsync(FlowEntry flowEntry) {
952         Status status;
953         if (isContainerModeAllowed(flowEntry)) {
954             status = removeEntry(flowEntry, true);
955         } else {
956             String msg = "Controller in container mode: Uninstall Refused";
957             status = new Status(StatusCode.NOTACCEPTABLE, msg);
958             log.warn(msg);
959         }
960         return status;
961     }
962
963     @Override
964     public Status modifyFlowEntry(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
965         Status status = null;
966         if (isContainerModeAllowed(currentFlowEntry)) {
967             status = modifyEntry(currentFlowEntry, newFlowEntry, false);
968         } else {
969             String msg = "Controller in container mode: Modify Refused";
970             String logMsg = msg + ": {}";
971             status = new Status(StatusCode.NOTACCEPTABLE, msg);
972             log.warn(logMsg, newFlowEntry);
973         }
974         return status;
975     }
976
977     @Override
978     public Status modifyFlowEntryAsync(FlowEntry currentFlowEntry, FlowEntry newFlowEntry) {
979         Status status = null;
980         if (isContainerModeAllowed(currentFlowEntry)) {
981             status = modifyEntry(currentFlowEntry, newFlowEntry, true);
982         } else {
983             String msg = "Controller in container mode: Modify Refused";
984             status = new Status(StatusCode.NOTACCEPTABLE, msg);
985             log.warn(msg);
986         }
987         return status;
988     }
989
990     /**
991      * Returns whether the specified flow entry is allowed to be
992      * installed/removed/modified based on the current container mode status.
993      * This call always returns true in the container instance of forwarding
994      * rules manager. It is meant for the global instance only (default
995      * container) of forwarding rules manager. Idea is that for assuring
996      * container isolation of traffic, flow installation in default container is
997      * blocked when in container mode (containers are present). The only flows
998      * that are allowed in container mode in the default container are the
999      * proactive flows, the ones automatically installed on the network node
1000      * which forwarding mode has been configured to "proactive". These flows are
1001      * needed by controller to discover the nodes topology and to discover the
1002      * attached hosts for some SDN switches.
1003      *
1004      * @param flowEntry
1005      *            The flow entry to be installed/removed/modified
1006      * @return true if not in container mode or if flowEntry is internally
1007      *         generated
1008      */
1009     private boolean isContainerModeAllowed(FlowEntry flowEntry) {
1010         return (!inContainerMode) ? true : flowEntry.isInternal();
1011     }
1012
1013     @Override
1014     public Status modifyOrAddFlowEntry(FlowEntry newFlowEntry) {
1015         /*
1016          * Run a check on the original entries to decide whether to go with a
1017          * add or modify method. A loose check means only check against the
1018          * original flow entry requests and not against the installed flow
1019          * entries which are the result of the original entry merged with the
1020          * container flow(s) (if any). The modifyFlowEntry method in presence of
1021          * conflicts with the Container flows (if any) would revert back to a
1022          * delete + add pattern
1023          */
1024         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1025
1026         if (currentFlowEntry != null) {
1027             return modifyFlowEntry(currentFlowEntry, newFlowEntry);
1028         } else {
1029             return installFlowEntry(newFlowEntry);
1030         }
1031     }
1032
1033     @Override
1034     public Status modifyOrAddFlowEntryAsync(FlowEntry newFlowEntry) {
1035         /*
1036          * Run a check on the original entries to decide whether to go with a
1037          * add or modify method. A loose check means only check against the
1038          * original flow entry requests and not against the installed flow
1039          * entries which are the result of the original entry merged with the
1040          * container flow(s) (if any). The modifyFlowEntry method in presence of
1041          * conflicts with the Container flows (if any) would revert back to a
1042          * delete + add pattern
1043          */
1044         FlowEntry currentFlowEntry = originalSwView.get(newFlowEntry);
1045
1046         if (currentFlowEntry != null) {
1047             return modifyFlowEntryAsync(currentFlowEntry, newFlowEntry);
1048         } else {
1049             return installFlowEntryAsync(newFlowEntry);
1050         }
1051     }
1052
1053     @Override
1054     public Status uninstallFlowEntryGroup(String groupName) {
1055         if (groupName == null || groupName.isEmpty()) {
1056             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1057         }
1058         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1059             return new Status(StatusCode.BADREQUEST, "Internal static flows group cannot be deleted through this api");
1060         }
1061         if (inContainerMode) {
1062             String msg = "Controller in container mode: Group Uninstall Refused";
1063             String logMsg = msg + ": {}";
1064             log.warn(logMsg, groupName);
1065             return new Status(StatusCode.NOTACCEPTABLE, msg);
1066         }
1067         int toBeRemoved = 0;
1068         String error = "";
1069         if (groupFlows.containsKey(groupName)) {
1070             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1071             toBeRemoved = list.size();
1072             for (FlowEntryInstall entry : list) {
1073                 Status status = this.removeEntry(entry.getOriginal(), false);
1074                 if (status.isSuccess()) {
1075                     toBeRemoved -= 1;
1076                 } else {
1077                     error = status.getDescription();
1078                 }
1079             }
1080         }
1081         return (toBeRemoved == 0) ? new Status(StatusCode.SUCCESS) : new Status(StatusCode.INTERNALERROR,
1082                 "Not all the flows were removed: " + error);
1083     }
1084
1085     @Override
1086     public Status uninstallFlowEntryGroupAsync(String groupName) {
1087         if (groupName == null || groupName.isEmpty()) {
1088             return new Status(StatusCode.BADREQUEST, "Invalid group name");
1089         }
1090         if (groupName.equals(FlowConfig.INTERNALSTATICFLOWGROUP)) {
1091             return new Status(StatusCode.BADREQUEST, "Static flows group cannot be deleted through this api");
1092         }
1093         if (inContainerMode) {
1094             String msg = "Controller in container mode: Group Uninstall Refused";
1095             String logMsg = msg + ": {}";
1096             log.warn(logMsg, groupName);
1097             return new Status(StatusCode.NOTACCEPTABLE, msg);
1098         }
1099         if (groupFlows.containsKey(groupName)) {
1100             List<FlowEntryInstall> list = new ArrayList<FlowEntryInstall>(groupFlows.get(groupName));
1101             for (FlowEntryInstall entry : list) {
1102                 this.removeEntry(entry.getOriginal(), true);
1103             }
1104         }
1105         return new Status(StatusCode.SUCCESS);
1106     }
1107
1108     @Override
1109     public boolean checkFlowEntryConflict(FlowEntry flowEntry) {
1110         return entryConflictsWithContainerFlows(flowEntry);
1111     }
1112
1113     /**
1114      * Updates all installed flows because the container flow got updated This
1115      * is obtained in two phases on per node basis: 1) Uninstall of all flows 2)
1116      * Reinstall of all flows This is needed because a new container flows
1117      * merged flow may conflict with an existing old container flows merged flow
1118      * on the network node
1119      */
1120     private void updateFlowsContainerFlow() {
1121         Set<FlowEntry> toReInstall = new HashSet<FlowEntry>();
1122         // First remove all installed entries
1123         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> entry : installedSwView.entrySet()) {
1124             FlowEntryInstall current = entry.getValue();
1125             // Store the original entry
1126             toReInstall.add(current.getOriginal());
1127             // Remove the old couples. No validity checks to be run, use the
1128             // internal remove
1129             this.removeEntryInternal(current, false);
1130         }
1131         // Then reinstall the original entries
1132         for (FlowEntry entry : toReInstall) {
1133             // Reinstall the original flow entries, via the regular path: new
1134             // cFlow merge + validations
1135             this.installFlowEntry(entry);
1136         }
1137     }
1138
1139     private void nonClusterObjectCreate() {
1140         originalSwView = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1141         installedSwView = new ConcurrentHashMap<FlowEntryInstall, FlowEntryInstall>();
1142         nodeFlows = new ConcurrentHashMap<Node, List<FlowEntryInstall>>();
1143         groupFlows = new ConcurrentHashMap<String, List<FlowEntryInstall>>();
1144         TSPolicies = new ConcurrentHashMap<String, Object>();
1145         staticFlowsOrdinal = new ConcurrentHashMap<Integer, Integer>();
1146         portGroupConfigs = new ConcurrentHashMap<String, PortGroupConfig>();
1147         portGroupData = new ConcurrentHashMap<PortGroupConfig, Map<Node, PortGroup>>();
1148         staticFlows = new ConcurrentHashMap<Integer, FlowConfig>();
1149         inactiveFlows = new ConcurrentHashMap<FlowEntry, FlowEntry>();
1150     }
1151
1152     private void registerWithOSGIConsole() {
1153         BundleContext bundleContext = FrameworkUtil.getBundle(this.getClass()).getBundleContext();
1154         bundleContext.registerService(CommandProvider.class.getName(), this, null);
1155     }
1156
1157     @Override
1158     public void setTSPolicyData(String policyname, Object o, boolean add) {
1159
1160         if (add) {
1161             /* Check if this policy already exists */
1162             if (!(TSPolicies.containsKey(policyname))) {
1163                 TSPolicies.put(policyname, o);
1164             }
1165         } else {
1166             TSPolicies.remove(policyname);
1167         }
1168         if (frmAware != null) {
1169             synchronized (frmAware) {
1170                 for (IForwardingRulesManagerAware frma : frmAware) {
1171                     try {
1172                         frma.policyUpdate(policyname, add);
1173                     } catch (Exception e) {
1174                         log.warn("Exception on callback", e);
1175                     }
1176                 }
1177             }
1178         }
1179     }
1180
1181     @Override
1182     public Map<String, Object> getTSPolicyData() {
1183         return TSPolicies;
1184     }
1185
1186     @Override
1187     public Object getTSPolicyData(String policyName) {
1188         if (TSPolicies.containsKey(policyName)) {
1189             return TSPolicies.get(policyName);
1190         } else {
1191             return null;
1192         }
1193     }
1194
1195     @Override
1196     public List<FlowEntry> getFlowEntriesForGroup(String policyName) {
1197         List<FlowEntry> list = new ArrayList<FlowEntry>();
1198         if (policyName != null && !policyName.trim().isEmpty()) {
1199             for (Map.Entry<FlowEntry, FlowEntry> entry : this.originalSwView.entrySet()) {
1200                 if (policyName.equals(entry.getKey().getGroupName())) {
1201                     list.add(entry.getKey().clone());
1202                 }
1203             }
1204         }
1205         return list;
1206     }
1207
1208     @Override
1209     public List<FlowEntry> getInstalledFlowEntriesForGroup(String policyName) {
1210         List<FlowEntry> list = new ArrayList<FlowEntry>();
1211         if (policyName != null && !policyName.trim().isEmpty()) {
1212             for (Map.Entry<FlowEntryInstall, FlowEntryInstall> entry : this.installedSwView.entrySet()) {
1213                 if (policyName.equals(entry.getKey().getGroupName())) {
1214                     list.add(entry.getKey().getInstall().clone());
1215                 }
1216             }
1217         }
1218         return list;
1219     }
1220
1221     @Override
1222     public void addOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1223
1224         for (FlowEntryInstall flow : this.nodeFlows.get(node)) {
1225             if (flow.getFlowName().equals(flowName)) {
1226                 FlowEntry currentFlowEntry = flow.getOriginal();
1227                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1228                 for (NodeConnector dstPort : portList) {
1229                     newFlowEntry.getFlow().addAction(new Output(dstPort));
1230                 }
1231                 Status error = modifyEntry(currentFlowEntry, newFlowEntry, false);
1232                 if (error.isSuccess()) {
1233                     log.info("Ports {} added to FlowEntry {}", portList, flowName);
1234                 } else {
1235                     log.warn("Failed to add ports {} to Flow entry {}. The failure is: {}", portList,
1236                             currentFlowEntry.toString(), error.getDescription());
1237                 }
1238                 return;
1239             }
1240         }
1241         log.warn("Failed to add ports to Flow {} on Node {}: Entry Not Found", flowName, node);
1242     }
1243
1244     @Override
1245     public void removeOutputPort(Node node, String flowName, List<NodeConnector> portList) {
1246         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1247             FlowEntryInstall flow = this.installedSwView.get(index);
1248             if (flow.getFlowName().equals(flowName)) {
1249                 FlowEntry currentFlowEntry = flow.getOriginal();
1250                 FlowEntry newFlowEntry = currentFlowEntry.clone();
1251                 for (NodeConnector dstPort : portList) {
1252                     Action action = new Output(dstPort);
1253                     newFlowEntry.getFlow().removeAction(action);
1254                 }
1255                 Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1256                 if (status.isSuccess()) {
1257                     log.info("Ports {} removed from FlowEntry {}", portList, flowName);
1258                 } else {
1259                     log.warn("Failed to remove ports {} from Flow entry {}. The failure is: {}", portList,
1260                             currentFlowEntry.toString(), status.getDescription());
1261                 }
1262                 return;
1263             }
1264         }
1265         log.warn("Failed to remove ports from Flow {} on Node {}: Entry Not Found", flowName, node);
1266     }
1267
1268     /*
1269      * This function assumes the target flow has only one output port
1270      */
1271     @Override
1272     public void replaceOutputPort(Node node, String flowName, NodeConnector outPort) {
1273         FlowEntry currentFlowEntry = null;
1274         FlowEntry newFlowEntry = null;
1275
1276         // Find the flow
1277         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1278             FlowEntryInstall flow = this.installedSwView.get(index);
1279             if (flow.getFlowName().equals(flowName)) {
1280                 currentFlowEntry = flow.getOriginal();
1281                 break;
1282             }
1283         }
1284         if (currentFlowEntry == null) {
1285             log.warn("Failed to replace output port for flow {} on node {}: Entry Not Found", flowName, node);
1286             return;
1287         }
1288
1289         // Create a flow copy with the new output port
1290         newFlowEntry = currentFlowEntry.clone();
1291         Action target = null;
1292         for (Action action : newFlowEntry.getFlow().getActions()) {
1293             if (action.getType() == ActionType.OUTPUT) {
1294                 target = action;
1295                 break;
1296             }
1297         }
1298         newFlowEntry.getFlow().removeAction(target);
1299         newFlowEntry.getFlow().addAction(new Output(outPort));
1300
1301         // Modify on network node
1302         Status status = modifyEntry(currentFlowEntry, newFlowEntry, false);
1303
1304         if (status.isSuccess()) {
1305             log.info("Output port replaced with {} for flow {} on node {}", outPort, flowName, node);
1306         } else {
1307             log.warn("Failed to replace output port for flow {} on node {}. The failure is: {}", flowName, node,
1308                     status.getDescription());
1309         }
1310         return;
1311     }
1312
1313     @Override
1314     public NodeConnector getOutputPort(Node node, String flowName) {
1315         for (FlowEntryInstall index : this.nodeFlows.get(node)) {
1316             FlowEntryInstall flow = this.installedSwView.get(index);
1317             if (flow.getFlowName().equals(flowName)) {
1318                 for (Action action : flow.getOriginal().getFlow().getActions()) {
1319                     if (action.getType() == ActionType.OUTPUT) {
1320                         return ((Output) action).getPort();
1321                     }
1322                 }
1323             }
1324         }
1325         return null;
1326     }
1327
1328     private void cacheStartup() {
1329         allocateCaches();
1330         retrieveCaches();
1331     }
1332
1333     private void allocateCaches() {
1334         if (this.clusterContainerService == null) {
1335             log.warn("Un-initialized clusterContainerService, can't create cache");
1336             return;
1337         }
1338
1339         log.debug("Allocating caches for Container {}", container.getName());
1340
1341         try {
1342             clusterContainerService.createCache("frm.originalSwView",
1343                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1344
1345             clusterContainerService.createCache("frm.installedSwView",
1346                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1347
1348             clusterContainerService.createCache("frm.inactiveFlows",
1349                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1350
1351             clusterContainerService.createCache("frm.nodeFlows",
1352                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1353
1354             clusterContainerService.createCache("frm.groupFlows",
1355                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1356
1357             clusterContainerService.createCache("frm.staticFlows",
1358                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1359
1360             clusterContainerService.createCache("frm.flowsSaveEvent",
1361                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1362
1363             clusterContainerService.createCache("frm.staticFlowsOrdinal",
1364                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1365
1366             clusterContainerService.createCache("frm.portGroupConfigs",
1367                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1368
1369             clusterContainerService.createCache("frm.portGroupData",
1370                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1371
1372             clusterContainerService.createCache("frm.TSPolicies",
1373                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
1374
1375             clusterContainerService.createCache(WORKSTATUSCACHE,
1376                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1377
1378             clusterContainerService.createCache(WORKORDERCACHE,
1379                     EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
1380
1381         } catch (CacheConfigException cce) {
1382             log.error("CacheConfigException");
1383         } catch (CacheExistException cce) {
1384             log.error("CacheExistException");
1385         }
1386     }
1387
1388     @SuppressWarnings({ "unchecked" })
1389     private void retrieveCaches() {
1390         ConcurrentMap<?, ?> map;
1391
1392         if (this.clusterContainerService == null) {
1393             log.warn("un-initialized clusterContainerService, can't retrieve cache");
1394             nonClusterObjectCreate();
1395             return;
1396         }
1397
1398         log.debug("Retrieving Caches for Container {}", container.getName());
1399
1400         map = clusterContainerService.getCache("frm.originalSwView");
1401         if (map != null) {
1402             originalSwView = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1403         } else {
1404             log.error("Retrieval of frm.originalSwView cache failed for Container {}", container.getName());
1405         }
1406
1407         map = clusterContainerService.getCache("frm.installedSwView");
1408         if (map != null) {
1409             installedSwView = (ConcurrentMap<FlowEntryInstall, FlowEntryInstall>) map;
1410         } else {
1411             log.error("Retrieval of frm.installedSwView cache failed for Container {}", container.getName());
1412         }
1413
1414         map = clusterContainerService.getCache("frm.inactiveFlows");
1415         if (map != null) {
1416             inactiveFlows = (ConcurrentMap<FlowEntry, FlowEntry>) map;
1417         } else {
1418             log.error("Retrieval of frm.inactiveFlows cache failed for Container {}", container.getName());
1419         }
1420
1421         map = clusterContainerService.getCache("frm.nodeFlows");
1422         if (map != null) {
1423             nodeFlows = (ConcurrentMap<Node, List<FlowEntryInstall>>) map;
1424         } else {
1425             log.error("Retrieval of cache failed for Container {}", container.getName());
1426         }
1427
1428         map = clusterContainerService.getCache("frm.groupFlows");
1429         if (map != null) {
1430             groupFlows = (ConcurrentMap<String, List<FlowEntryInstall>>) map;
1431         } else {
1432             log.error("Retrieval of frm.groupFlows cache failed for Container {}", container.getName());
1433         }
1434
1435         map = clusterContainerService.getCache("frm.staticFlows");
1436         if (map != null) {
1437             staticFlows = (ConcurrentMap<Integer, FlowConfig>) map;
1438         } else {
1439             log.error("Retrieval of frm.staticFlows cache failed for Container {}", container.getName());
1440         }
1441
1442         map = clusterContainerService.getCache("frm.staticFlowsOrdinal");
1443         if (map != null) {
1444             staticFlowsOrdinal = (ConcurrentMap<Integer, Integer>) map;
1445         } else {
1446             log.error("Retrieval of frm.staticFlowsOrdinal cache failed for Container {}", container.getName());
1447         }
1448
1449         map = clusterContainerService.getCache("frm.portGroupConfigs");
1450         if (map != null) {
1451             portGroupConfigs = (ConcurrentMap<String, PortGroupConfig>) map;
1452         } else {
1453             log.error("Retrieval of frm.portGroupConfigs cache failed for Container {}", container.getName());
1454         }
1455
1456         map = clusterContainerService.getCache("frm.portGroupData");
1457         if (map != null) {
1458             portGroupData = (ConcurrentMap<PortGroupConfig, Map<Node, PortGroup>>) map;
1459         } else {
1460             log.error("Retrieval of frm.portGroupData allocation failed for Container {}", container.getName());
1461         }
1462
1463         map = clusterContainerService.getCache("frm.TSPolicies");
1464         if (map != null) {
1465             TSPolicies = (ConcurrentMap<String, Object>) map;
1466         } else {
1467             log.error("Retrieval of frm.TSPolicies cache failed for Container {}", container.getName());
1468         }
1469
1470         map = clusterContainerService.getCache(WORKORDERCACHE);
1471         if (map != null) {
1472             workOrder = (ConcurrentMap<FlowEntryDistributionOrder, FlowEntryInstall>) map;
1473         } else {
1474             log.error("Retrieval of " + WORKORDERCACHE + " cache failed for Container {}", container.getName());
1475         }
1476
1477         map = clusterContainerService.getCache(WORKSTATUSCACHE);
1478         if (map != null) {
1479             workStatus = (ConcurrentMap<FlowEntryDistributionOrder, Status>) map;
1480         } else {
1481             log.error("Retrieval of " + WORKSTATUSCACHE + " cache failed for Container {}", container.getName());
1482         }
1483     }
1484
1485     private boolean flowConfigExists(FlowConfig config) {
1486         // Flow name has to be unique on per node id basis
1487         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1488             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1489                 return true;
1490             }
1491         }
1492         return false;
1493     }
1494
1495     @Override
1496     public Status addStaticFlow(FlowConfig config) {
1497         // Configuration object validation
1498         Status status = config.validate(container);
1499         if (!status.isSuccess()) {
1500             log.warn("Invalid Configuration for flow {}. The failure is {}", config, status.getDescription());
1501             String error = "Invalid Configuration (" + status.getDescription() + ")";
1502             config.setStatus(error);
1503             return new Status(StatusCode.BADREQUEST, error);
1504         }
1505         return addStaticFlowInternal(config, false);
1506     }
1507
1508     /**
1509      * Private method to add a static flow configuration which does not run any
1510      * validation on the passed FlowConfig object. If restore is set to true,
1511      * configuration is stored in configuration database regardless the
1512      * installation on the network node was successful. This is useful at boot
1513      * when static flows are present in startup configuration and are read
1514      * before the switches connects.
1515      *
1516      * @param config
1517      *            The static flow configuration
1518      * @param restore
1519      *            if true, the configuration is stored regardless the
1520      *            installation on the network node was successful
1521      * @return The status of this request
1522      */
1523     private Status addStaticFlowInternal(FlowConfig config, boolean restore) {
1524         boolean multipleFlowPush = false;
1525         String error;
1526         Status status;
1527         config.setStatus(SUCCESS);
1528
1529         // Presence check
1530         if (flowConfigExists(config)) {
1531             error = "Entry with this name on specified switch already exists";
1532             log.warn("Entry with this name on specified switch already exists: {}", config);
1533             config.setStatus(error);
1534             return new Status(StatusCode.CONFLICT, error);
1535         }
1536
1537         if ((config.getIngressPort() == null) && config.getPortGroup() != null) {
1538             for (String portGroupName : portGroupConfigs.keySet()) {
1539                 if (portGroupName.equalsIgnoreCase(config.getPortGroup())) {
1540                     multipleFlowPush = true;
1541                     break;
1542                 }
1543             }
1544             if (!multipleFlowPush) {
1545                 log.warn("Invalid Configuration(Invalid PortGroup Name) for flow {}", config);
1546                 error = "Invalid Configuration (Invalid PortGroup Name)";
1547                 config.setStatus(error);
1548                 return new Status(StatusCode.BADREQUEST, error);
1549             }
1550         }
1551
1552         /*
1553          * If requested program the entry in hardware first before updating the
1554          * StaticFlow DB
1555          */
1556         if (!multipleFlowPush) {
1557             // Program hw
1558             if (config.installInHw()) {
1559                 FlowEntry entry = config.getFlowEntry();
1560                 status = this.installFlowEntry(entry);
1561                 if (!status.isSuccess()) {
1562                     config.setStatus(status.getDescription());
1563                     if (!restore) {
1564                         return status;
1565                     }
1566                 }
1567             }
1568         }
1569
1570         /*
1571          * When the control reaches this point, either of the following
1572          * conditions is true 1. This is a single entry configuration (non
1573          * PortGroup) and the hardware installation is successful 2. This is a
1574          * multiple entry configuration (PortGroup) and hardware installation is
1575          * NOT done directly on this event. 3. The User prefers to retain the
1576          * configuration in Controller and skip hardware installation.
1577          *
1578          * Hence it is safe to update the StaticFlow DB at this point.
1579          *
1580          * Note : For the case of PortGrouping, it is essential to have this DB
1581          * populated before the PortGroupListeners can query for the DB
1582          * triggered using portGroupChanged event...
1583          */
1584         Integer ordinal = staticFlowsOrdinal.get(0);
1585         staticFlowsOrdinal.put(0, ++ordinal);
1586         staticFlows.put(ordinal, config);
1587
1588         if (multipleFlowPush) {
1589             PortGroupConfig pgconfig = portGroupConfigs.get(config.getPortGroup());
1590             Map<Node, PortGroup> existingData = portGroupData.get(pgconfig);
1591             if (existingData != null) {
1592                 portGroupChanged(pgconfig, existingData, true);
1593             }
1594         }
1595         return new Status(StatusCode.SUCCESS);
1596     }
1597
1598     private void addStaticFlowsToSwitch(Node node) {
1599         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1600             FlowConfig config = entry.getValue();
1601             if (config.isPortGroupEnabled()) {
1602                 continue;
1603             }
1604             if (config.getNode().equals(node)) {
1605                 if (config.installInHw() && !config.getStatus().equals(SUCCESS)) {
1606                     Status status = this.installFlowEntryAsync(config.getFlowEntry());
1607                     config.setStatus(status.getDescription());
1608                 }
1609             }
1610         }
1611         // Update cluster cache
1612         refreshClusterStaticFlowsStatus(node);
1613     }
1614
1615     private void updateStaticFlowConfigsOnNodeDown(Node node) {
1616         log.trace("Updating Static Flow configs on node down: {}", node);
1617
1618         List<Integer> toRemove = new ArrayList<Integer>();
1619         for (Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1620
1621             FlowConfig config = entry.getValue();
1622
1623             if (config.isPortGroupEnabled()) {
1624                 continue;
1625             }
1626
1627             if (config.installInHw() && config.getNode().equals(node)) {
1628                 if (config.isInternalFlow()) {
1629                     // Take note of this controller generated static flow
1630                     toRemove.add(entry.getKey());
1631                 } else {
1632                     config.setStatus(NODEDOWN);
1633                 }
1634             }
1635         }
1636         // Remove controller generated static flows for this node
1637         for (Integer index : toRemove) {
1638             staticFlows.remove(index);
1639         }
1640         // Update cluster cache
1641         refreshClusterStaticFlowsStatus(node);
1642
1643     }
1644
1645     private void updateStaticFlowConfigsOnContainerModeChange(UpdateType update) {
1646         log.trace("Updating Static Flow configs on container mode change: {}", update);
1647
1648         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1649             FlowConfig config = entry.getValue();
1650             if (config.isPortGroupEnabled()) {
1651                 continue;
1652             }
1653             if (config.installInHw() && !config.isInternalFlow()) {
1654                 switch (update) {
1655                 case ADDED:
1656                     config.setStatus("Removed from node because in container mode");
1657                     break;
1658                 case REMOVED:
1659                     config.setStatus(SUCCESS);
1660                     break;
1661                 default:
1662                 }
1663             }
1664         }
1665         // Update cluster cache
1666         refreshClusterStaticFlowsStatus(null);
1667     }
1668
1669     @Override
1670     public Status removeStaticFlow(FlowConfig config) {
1671         /*
1672          * No config.isInternal() check as NB does not take this path and GUI
1673          * cannot issue a delete on an internal generated flow. We need this
1674          * path to be accessible when switch mode is changed from proactive to
1675          * reactive, so that we can remove the internal generated LLDP and ARP
1676          * punt flows
1677          */
1678
1679         // Look for the target configuration entry
1680         Integer key = 0;
1681         FlowConfig target = null;
1682         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1683             if (entry.getValue().isByNameAndNodeIdEqual(config)) {
1684                 key = entry.getKey();
1685                 target = entry.getValue();
1686                 break;
1687             }
1688         }
1689         if (target == null) {
1690             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1691         }
1692
1693         // Program the network node
1694         Status status = this.uninstallFlowEntry(config.getFlowEntry());
1695
1696         // Update configuration database if programming was successful
1697         if (status.isSuccess()) {
1698             staticFlows.remove(key);
1699         }
1700
1701         return status;
1702     }
1703
1704     @Override
1705     public Status removeStaticFlow(String name, Node node) {
1706         // Look for the target configuration entry
1707         Integer key = 0;
1708         FlowConfig target = null;
1709         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1710             if (mapEntry.getValue().isByNameAndNodeIdEqual(name, node)) {
1711                 key = mapEntry.getKey();
1712                 target = mapEntry.getValue();
1713                 break;
1714             }
1715         }
1716         if (target == null) {
1717             return new Status(StatusCode.NOTFOUND, "Entry Not Present");
1718         }
1719
1720         // Validity check for api3 entry point
1721         if (target.isInternalFlow()) {
1722             String msg = "Invalid operation: Controller generated flow cannot be deleted";
1723             String logMsg = msg + ": {}";
1724             log.warn(logMsg, name);
1725             return new Status(StatusCode.NOTACCEPTABLE, msg);
1726         }
1727
1728         if (target.isPortGroupEnabled()) {
1729             String msg = "Invalid operation: Port Group flows cannot be deleted through this API";
1730             String logMsg = msg + ": {}";
1731             log.warn(logMsg, name);
1732             return new Status(StatusCode.NOTACCEPTABLE, msg);
1733         }
1734
1735         // Program the network node
1736         Status status = this.removeEntry(target.getFlowEntry(), false);
1737
1738         // Update configuration database if programming was successful
1739         if (status.isSuccess()) {
1740             staticFlows.remove(key);
1741         }
1742
1743         return status;
1744     }
1745
1746     @Override
1747     public Status modifyStaticFlow(FlowConfig newFlowConfig) {
1748         // Validity check for api3 entry point
1749         if (newFlowConfig.isInternalFlow()) {
1750             String msg = "Invalid operation: Controller generated flow cannot be modified";
1751             String logMsg = msg + ": {}";
1752             log.warn(logMsg, newFlowConfig);
1753             return new Status(StatusCode.NOTACCEPTABLE, msg);
1754         }
1755
1756         // Validity Check
1757         Status status = newFlowConfig.validate(container);
1758         if (!status.isSuccess()) {
1759             String msg = "Invalid Configuration (" + status.getDescription() + ")";
1760             newFlowConfig.setStatus(msg);
1761             log.warn("Invalid Configuration for flow {}. The failure is {}", newFlowConfig, status.getDescription());
1762             return new Status(StatusCode.BADREQUEST, msg);
1763         }
1764
1765         FlowConfig oldFlowConfig = null;
1766         Integer index = null;
1767         for (ConcurrentMap.Entry<Integer, FlowConfig> mapEntry : staticFlows.entrySet()) {
1768             FlowConfig entry = mapEntry.getValue();
1769             if (entry.isByNameAndNodeIdEqual(newFlowConfig.getName(), newFlowConfig.getNode())) {
1770                 oldFlowConfig = entry;
1771                 index = mapEntry.getKey();
1772                 break;
1773             }
1774         }
1775
1776         if (oldFlowConfig == null) {
1777             String msg = "Attempt to modify a non existing static flow";
1778             String logMsg = msg + ": {}";
1779             log.warn(logMsg, newFlowConfig);
1780             return new Status(StatusCode.NOTFOUND, msg);
1781         }
1782
1783         // Do not attempt to reinstall the flow, warn user
1784         if (newFlowConfig.equals(oldFlowConfig)) {
1785             String msg = "No modification detected";
1786             log.info("Static flow modification skipped. New flow and old flow are the same: {}", newFlowConfig);
1787             return new Status(StatusCode.SUCCESS, msg);
1788         }
1789
1790         // If flow is installed, program the network node
1791         status = new Status(StatusCode.SUCCESS, "Saved in config");
1792         if (oldFlowConfig.installInHw()) {
1793             status = this.modifyFlowEntry(oldFlowConfig.getFlowEntry(), newFlowConfig.getFlowEntry());
1794         }
1795
1796         // Update configuration database if programming was successful
1797         if (status.isSuccess()) {
1798             newFlowConfig.setStatus(status.getDescription());
1799             staticFlows.put(index, newFlowConfig);
1800         }
1801
1802         return status;
1803     }
1804
1805     @Override
1806     public Status toggleStaticFlowStatus(String name, Node node) {
1807         return toggleStaticFlowStatus(getStaticFlow(name, node));
1808     }
1809
1810     @Override
1811     public Status toggleStaticFlowStatus(FlowConfig config) {
1812         if (config == null) {
1813             String msg = "Invalid request: null flow config";
1814             log.warn(msg);
1815             return new Status(StatusCode.BADREQUEST, msg);
1816         }
1817         // Validity check for api3 entry point
1818         if (config.isInternalFlow()) {
1819             String msg = "Invalid operation: Controller generated flow cannot be modified";
1820             String logMsg = msg + ": {}";
1821             log.warn(logMsg, config);
1822             return new Status(StatusCode.NOTACCEPTABLE, msg);
1823         }
1824
1825         // Find the config entry
1826         Integer key = 0;
1827         FlowConfig target = null;
1828         for (Map.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1829             FlowConfig conf = entry.getValue();
1830             if (conf.isByNameAndNodeIdEqual(config)) {
1831                 key = entry.getKey();
1832                 target = conf;
1833                 break;
1834             }
1835         }
1836         if (target != null) {
1837             Status status = target.validate(container);
1838             if (!status.isSuccess()) {
1839                 log.warn(status.getDescription());
1840                 return status;
1841             }
1842             status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
1843                                     .installFlowEntry(target.getFlowEntry());
1844             if (status.isSuccess()) {
1845                 // Update Configuration database
1846                 target.setStatus(SUCCESS);
1847                 target.toggleInstallation();
1848                 staticFlows.put(key, target);
1849             }
1850             return status;
1851         }
1852
1853         return new Status(StatusCode.NOTFOUND, "Unable to locate the entry. Failed to toggle status");
1854     }
1855
1856     /**
1857      * Reinsert all static flows entries in the cache to force cache updates in
1858      * the cluster. This is useful when only some parameters were changed in the
1859      * entries, like the status.
1860      *
1861      * @param node
1862      *            The node for which the static flow configurations have to be
1863      *            refreshed. If null, all nodes static flows will be refreshed.
1864      */
1865     private void refreshClusterStaticFlowsStatus(Node node) {
1866         // Refresh cluster cache
1867         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1868             if (node == null || entry.getValue().getNode().equals(node)) {
1869                 staticFlows.put(entry.getKey(), entry.getValue());
1870             }
1871         }
1872     }
1873
1874     /**
1875      * Uninstall all the non-internal Flow Entries present in the software view.
1876      * If requested, a copy of each original flow entry will be stored in the
1877      * inactive list so that it can be re-applied when needed (This is typically
1878      * the case when running in the default container and controller moved to
1879      * container mode)
1880      *
1881      * @param preserveFlowEntries
1882      *            if true, a copy of each original entry is stored in the
1883      *            inactive list
1884      */
1885     private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
1886         log.info("Uninstalling all non-internal flows");
1887
1888         List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
1889
1890         // Store entries / create target list
1891         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
1892             FlowEntryInstall flowEntries = mapEntry.getValue();
1893             // Skip internal generated static flows
1894             if (!flowEntries.isInternal()) {
1895                 toRemove.add(flowEntries);
1896                 // Store the original entries if requested
1897                 if (preserveFlowEntries) {
1898                     inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
1899                 }
1900             }
1901         }
1902
1903         // Now remove the entries
1904         for (FlowEntryInstall flowEntryHw : toRemove) {
1905             Status status = this.removeEntryInternal(flowEntryHw, false);
1906             if (!status.isSuccess()) {
1907                 log.warn("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
1908             }
1909         }
1910     }
1911
1912     /**
1913      * Re-install all the Flow Entries present in the inactive list The inactive
1914      * list will be empty at the end of this call This function is called on the
1915      * default container instance of FRM only when the last container is deleted
1916      */
1917     private void reinstallAllFlowEntries() {
1918         log.info("Reinstalling all inactive flows");
1919
1920         for (FlowEntry flowEntry : this.inactiveFlows.keySet()) {
1921             this.addEntry(flowEntry, false);
1922         }
1923
1924         // Empty inactive list in any case
1925         inactiveFlows.clear();
1926     }
1927
1928     @Override
1929     public List<FlowConfig> getStaticFlows() {
1930         return getStaticFlowsOrderedList(staticFlows, staticFlowsOrdinal.get(0).intValue());
1931     }
1932
1933     // TODO: need to come out with a better algorithm for maintaining the order
1934     // of the configuration entries
1935     // with actual one, index associated to deleted entries cannot be reused and
1936     // map grows...
1937     private List<FlowConfig> getStaticFlowsOrderedList(ConcurrentMap<Integer, FlowConfig> flowMap, int maxKey) {
1938         List<FlowConfig> orderedList = new ArrayList<FlowConfig>();
1939         for (int i = 0; i <= maxKey; i++) {
1940             FlowConfig entry = flowMap.get(i);
1941             if (entry != null) {
1942                 orderedList.add(entry);
1943             }
1944         }
1945         return orderedList;
1946     }
1947
1948     @Override
1949     public FlowConfig getStaticFlow(String name, Node node) {
1950         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1951             if (entry.getValue().isByNameAndNodeIdEqual(name, node)) {
1952                 return entry.getValue();
1953             }
1954         }
1955         return null;
1956     }
1957
1958     @Override
1959     public List<FlowConfig> getStaticFlows(Node node) {
1960         List<FlowConfig> list = new ArrayList<FlowConfig>();
1961         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1962             if (entry.getValue().onNode(node)) {
1963                 list.add(entry.getValue());
1964             }
1965         }
1966         return list;
1967     }
1968
1969     @Override
1970     public List<String> getStaticFlowNamesForNode(Node node) {
1971         List<String> list = new ArrayList<String>();
1972         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1973             if (entry.getValue().onNode(node)) {
1974                 list.add(entry.getValue().getName());
1975             }
1976         }
1977         return list;
1978     }
1979
1980     @Override
1981     public List<Node> getListNodeWithConfiguredFlows() {
1982         Set<Node> set = new HashSet<Node>();
1983         for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
1984             set.add(entry.getValue().getNode());
1985         }
1986         return new ArrayList<Node>(set);
1987     }
1988
1989     @SuppressWarnings("unchecked")
1990     private void loadFlowConfiguration() {
1991         ObjectReader objReader = new ObjectReader();
1992         ConcurrentMap<Integer, FlowConfig> confList = (ConcurrentMap<Integer, FlowConfig>) objReader.read(this,
1993                 frmFileName);
1994
1995         ConcurrentMap<String, PortGroupConfig> pgConfig = (ConcurrentMap<String, PortGroupConfig>) objReader.read(this,
1996                 portGroupFileName);
1997
1998         if (pgConfig != null) {
1999             for (ConcurrentMap.Entry<String, PortGroupConfig> entry : pgConfig.entrySet()) {
2000                 addPortGroupConfig(entry.getKey(), entry.getValue().getMatchString(), true);
2001             }
2002         }
2003
2004         if (confList == null) {
2005             return;
2006         }
2007
2008         int maxKey = 0;
2009         for (Integer key : confList.keySet()) {
2010             if (key.intValue() > maxKey) {
2011                 maxKey = key.intValue();
2012             }
2013         }
2014
2015         for (FlowConfig conf : getStaticFlowsOrderedList(confList, maxKey)) {
2016             addStaticFlowInternal(conf, true);
2017         }
2018     }
2019
2020     @Override
2021     public Object readObject(ObjectInputStream ois) throws FileNotFoundException, IOException, ClassNotFoundException {
2022         return ois.readObject();
2023     }
2024
2025     @Override
2026     public Status saveConfig() {
2027         return saveConfigInternal();
2028     }
2029
2030     private Status saveConfigInternal() {
2031         ObjectWriter objWriter = new ObjectWriter();
2032         ConcurrentMap<Integer, FlowConfig> nonDynamicFlows = new ConcurrentHashMap<Integer, FlowConfig>();
2033         for (Integer ordinal : staticFlows.keySet()) {
2034             FlowConfig config = staticFlows.get(ordinal);
2035             // Do not save dynamic and controller generated static flows
2036             if (config.isDynamic() || config.isInternalFlow()) {
2037                 continue;
2038             }
2039             nonDynamicFlows.put(ordinal, config);
2040         }
2041         objWriter.write(nonDynamicFlows, frmFileName);
2042         objWriter.write(new ConcurrentHashMap<String, PortGroupConfig>(portGroupConfigs), portGroupFileName);
2043         return new Status(StatusCode.SUCCESS, null);
2044     }
2045
2046     @Override
2047     public void subnetNotify(Subnet sub, boolean add) {
2048     }
2049
2050     private void installImplicitARPReplyPunt(Node node) {
2051
2052         if (node == null) {
2053             return;
2054         }
2055
2056         List<String> puntAction = new ArrayList<String>();
2057         puntAction.add(ActionType.CONTROLLER.toString());
2058
2059         FlowConfig allowARP = new FlowConfig();
2060         allowARP.setInstallInHw(true);
2061         allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP Reply" + FlowConfig.INTERNALSTATICFLOWEND);
2062         allowARP.setPriority("500");
2063         allowARP.setNode(node);
2064         allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue()).toUpperCase());
2065         allowARP.setDstMac(HexEncode.bytesToHexString(switchManager.getControllerMAC()));
2066         allowARP.setActions(puntAction);
2067         addStaticFlowInternal(allowARP, true); // skip validation on internal static flow name
2068     }
2069
2070     /**
2071      * (non-Javadoc)
2072      *
2073      * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
2074      *      boolean)
2075      *
2076      *      This method can be called from within the OSGi framework context,
2077      *      given the programming operation can take sometime, it not good
2078      *      pratice to have in it's context operations that can take time,
2079      *      hence moving off to a different thread for async processing.
2080      */
2081     private ExecutorService executor;
2082     @Override
2083     public void modeChangeNotify(final Node node, final boolean proactive) {
2084         Callable<Status> modeChangeCallable = new Callable<Status>() {
2085             @Override
2086             public Status call() throws Exception {
2087                 List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
2088
2089                 List<String> puntAction = new ArrayList<String>();
2090                 puntAction.add(ActionType.CONTROLLER.toString());
2091
2092                 FlowConfig allowARP = new FlowConfig();
2093                 allowARP.setInstallInHw(true);
2094                 allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
2095                 allowARP.setPriority("1");
2096                 allowARP.setNode(node);
2097                 allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
2098                         .toUpperCase());
2099                 allowARP.setActions(puntAction);
2100                 defaultConfigs.add(allowARP);
2101
2102                 FlowConfig allowLLDP = new FlowConfig();
2103                 allowLLDP.setInstallInHw(true);
2104                 allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
2105                 allowLLDP.setPriority("1");
2106                 allowLLDP.setNode(node);
2107                 allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
2108                         .toUpperCase());
2109                 allowLLDP.setActions(puntAction);
2110                 defaultConfigs.add(allowLLDP);
2111
2112                 List<String> dropAction = new ArrayList<String>();
2113                 dropAction.add(ActionType.DROP.toString());
2114
2115                 FlowConfig dropAllConfig = new FlowConfig();
2116                 dropAllConfig.setInstallInHw(true);
2117                 dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
2118                         + FlowConfig.INTERNALSTATICFLOWEND);
2119                 dropAllConfig.setPriority("0");
2120                 dropAllConfig.setNode(node);
2121                 dropAllConfig.setActions(dropAction);
2122                 defaultConfigs.add(dropAllConfig);
2123
2124                 log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
2125                 for (FlowConfig fc : defaultConfigs) {
2126                     Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
2127                     if (status.isSuccess()) {
2128                         log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
2129                     } else {
2130                         log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
2131                                 fc.getName());
2132                     }
2133                 }
2134                 return new Status(StatusCode.SUCCESS);
2135             }
2136         };
2137
2138         /*
2139          * Execute the work outside the caller context, this could be an
2140          * expensive operation and we don't want to block the caller for it.
2141          */
2142         this.executor.submit(modeChangeCallable);
2143     }
2144
2145     /**
2146      * Remove from the databases all the flows installed on the node
2147      *
2148      * @param node
2149      */
2150     private void cleanDatabaseForNode(Node node) {
2151         log.info("Cleaning Flow database for Node {}", node);
2152         if (nodeFlows.containsKey(node)) {
2153             List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>(nodeFlows.get(node));
2154
2155             for (FlowEntryInstall entry : toRemove) {
2156                 updateLocalDatabase(entry, false);
2157             }
2158         }
2159     }
2160
2161     private boolean doesFlowContainNodeConnector(Flow flow, NodeConnector nc) {
2162         if (nc == null) {
2163             return false;
2164         }
2165
2166         Match match = flow.getMatch();
2167         if (match.isPresent(MatchType.IN_PORT)) {
2168             NodeConnector matchPort = (NodeConnector) match.getField(MatchType.IN_PORT).getValue();
2169             if (matchPort.equals(nc)) {
2170                 return true;
2171             }
2172         }
2173         List<Action> actionsList = flow.getActions();
2174         if (actionsList != null) {
2175             for (Action action : actionsList) {
2176                 if (action instanceof Output) {
2177                     NodeConnector actionPort = ((Output) action).getPort();
2178                     if (actionPort.equals(nc)) {
2179                         return true;
2180                     }
2181                 }
2182             }
2183         }
2184         return false;
2185     }
2186
2187     @Override
2188     public void notifyNode(Node node, UpdateType type, Map<String, Property> propMap) {
2189         this.pendingEvents.offer(new NodeUpdateEvent(type, node));
2190     }
2191
2192     @Override
2193     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
2194         boolean updateStaticFlowCluster = false;
2195
2196         switch (type) {
2197         case ADDED:
2198             break;
2199         case CHANGED:
2200             Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
2201             if (config != null) {
2202                 switch (config.getValue()) {
2203                 case Config.ADMIN_DOWN:
2204                     log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
2205                     updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2206                     break;
2207                 case Config.ADMIN_UP:
2208                     log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
2209                     updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
2210                     break;
2211                 case Config.ADMIN_UNDEF:
2212                     break;
2213                 default:
2214                 }
2215             }
2216             break;
2217         case REMOVED:
2218             // This is the case where a switch port is removed from the SDN agent space
2219             log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
2220             updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
2221             break;
2222         default:
2223
2224         }
2225
2226         if (updateStaticFlowCluster) {
2227             refreshClusterStaticFlowsStatus(nodeConnector.getNode());
2228         }
2229     }
2230
2231     /*
2232      * It goes through the static flows configuration, it identifies the ones
2233      * which have the specified node connector as input or output port and
2234      * install them on the network node if they are marked to be installed in
2235      * hardware and their status shows they were not installed yet
2236      */
2237     private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
2238         boolean updated = false;
2239         List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
2240         for (FlowConfig flowConfig : flowConfigForNode) {
2241             if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
2242                 if (flowConfig.installInHw() && !flowConfig.getStatus().equals(SUCCESS)) {
2243                     Status status = this.installFlowEntry(flowConfig.getFlowEntry());
2244                     if (!status.isSuccess()) {
2245                         flowConfig.setStatus(status.getDescription());
2246                     } else {
2247                         flowConfig.setStatus(SUCCESS);
2248                     }
2249                     updated = true;
2250                 }
2251             }
2252         }
2253         return updated;
2254     }
2255
2256     /*
2257      * Remove from the network node all the flows which have the specified node
2258      * connector as input or output port. If any of the flow entry is a static
2259      * flow, it updates the correspondent configuration.
2260      */
2261     private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
2262         boolean updated = false;
2263         List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
2264         if (nodeFlowEntries == null) {
2265             return updated;
2266         }
2267         for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
2268             if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
2269                 Status status = this.removeEntryInternal(fei, true);
2270                 if (!status.isSuccess()) {
2271                     continue;
2272                 }
2273                 /*
2274                  * If the flow entry is a static flow, then update its
2275                  * configuration
2276                  */
2277                 if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
2278                     FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
2279                     if (flowConfig != null) {
2280                         flowConfig.setStatus(PORTREMOVED);
2281                         updated = true;
2282                     }
2283                 }
2284             }
2285         }
2286         return updated;
2287     }
2288
2289     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
2290         FlowConfig derivedFlow = new FlowConfig(original);