Merge "Introduce ASYNC caches and use them in FRM"
[controller.git] / opendaylight / forwardingrulesmanager / implementation / src / main / java / org / opendaylight / controller / forwardingrulesmanager / internal / ForwardingRulesManager.java
index 0d0874990e88a5902e510a62ca31aa1294c3d67d..3c918ec0e1c5dac1bf503737cd88c8681a61c77a 100644 (file)
@@ -22,10 +22,12 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.eclipse.osgi.framework.console.CommandInterpreter;
@@ -36,6 +38,7 @@ import org.opendaylight.controller.clustering.services.ICacheUpdateAware;
 import org.opendaylight.controller.clustering.services.IClusterContainerServices;
 import org.opendaylight.controller.clustering.services.IClusterServices;
 import org.opendaylight.controller.configuration.IConfigurationContainerAware;
+import org.opendaylight.controller.sal.connection.ConnectionLocality;
 import org.opendaylight.controller.connectionmanager.IConnectionManager;
 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
 import org.opendaylight.controller.forwardingrulesmanager.FlowEntry;
@@ -53,6 +56,7 @@ import org.opendaylight.controller.sal.action.Controller;
 import org.opendaylight.controller.sal.action.Flood;
 import org.opendaylight.controller.sal.action.Output;
 import org.opendaylight.controller.sal.action.PopVlan;
+import org.opendaylight.controller.sal.core.Config;
 import org.opendaylight.controller.sal.core.ContainerFlow;
 import org.opendaylight.controller.sal.core.IContainer;
 import org.opendaylight.controller.sal.core.IContainerListener;
@@ -98,7 +102,7 @@ public class ForwardingRulesManager implements
         IConfigurationContainerAware,
         IInventoryListener,
         IObjectReader,
-        ICacheUpdateAware,
+        ICacheUpdateAware<Object,Object>,
         CommandProvider,
         IFlowProgrammerListener {
     private static final String NODEDOWN = "Node is Down";
@@ -209,7 +213,8 @@ public class ForwardingRulesManager implements
      * @return a Future object for monitoring the progress of the result, or
      *         null in case the processing should take place locally
      */
-    private Future<Status> distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u, UpdateType t) {
+    private FlowEntryDistributionOrderFutureTask distributeWorkOrder(FlowEntryInstall e, FlowEntryInstall u,
+            UpdateType t) {
         // A null entry it's an unexpected condition, anyway it's safe to keep
         // the handling local
         if (e == null) {
@@ -217,7 +222,7 @@ public class ForwardingRulesManager implements
         }
 
         Node n = e.getNode();
-        if (!connectionManager.isLocal(n)) {
+        if (connectionManager.getLocalityStatus(n) == ConnectionLocality.NOT_LOCAL) {
             // Create the work order and distribute it
             FlowEntryDistributionOrder fe =
                     new FlowEntryDistributionOrder(e, t, clusterContainerService.getMyAddress());
@@ -237,8 +242,7 @@ public class ForwardingRulesManager implements
             return ret;
         }
 
-        logsync.trace("LOCAL Node {} so processing Entry:{} UpdateType:{}", n, e, t);
-
+        logsync.trace("Node {} could be local. so processing Entry:{} UpdateType:{}", n, e, t);
         return null;
     }
 
@@ -540,11 +544,17 @@ public class ForwardingRulesManager implements
      *         contain the unique id assigned to this request
      */
     private Status modifyEntryInternal(FlowEntryInstall currentEntries, FlowEntryInstall newEntries, boolean async) {
-        Future<Status> futureStatus = distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
+        FlowEntryDistributionOrderFutureTask futureStatus =
+                distributeWorkOrder(currentEntries, newEntries, UpdateType.CHANGED);
         if (futureStatus != null) {
             Status retStatus = new Status(StatusCode.UNDEFINED);
             try {
                 retStatus = futureStatus.get();
+                if (retStatus.getCode()
+                        .equals(StatusCode.TIMEOUT)) {
+                    // A timeout happened, lets cleanup the workMonitor
+                    workMonitor.remove(futureStatus.getOrder());
+                }
             } catch (InterruptedException e) {
                 log.error("", e);
             } catch (ExecutionException e) {
@@ -652,11 +662,16 @@ public class ForwardingRulesManager implements
      *         contain the unique id assigned to this request
      */
     private Status removeEntryInternal(FlowEntryInstall entry, boolean async) {
-        Future<Status> futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
+        FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.REMOVED);
         if (futureStatus != null) {
             Status retStatus = new Status(StatusCode.UNDEFINED);
             try {
                 retStatus = futureStatus.get();
+                if (retStatus.getCode()
+                        .equals(StatusCode.TIMEOUT)) {
+                    // A timeout happened, lets cleanup the workMonitor
+                    workMonitor.remove(futureStatus.getOrder());
+                }
             } catch (InterruptedException e) {
                 log.error("", e);
             } catch (ExecutionException e) {
@@ -700,11 +715,16 @@ public class ForwardingRulesManager implements
      *         contain the unique id assigned to this request
      */
     private Status addEntriesInternal(FlowEntryInstall entry, boolean async) {
-        Future<Status> futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
+        FlowEntryDistributionOrderFutureTask futureStatus = distributeWorkOrder(entry, null, UpdateType.ADDED);
         if (futureStatus != null) {
             Status retStatus = new Status(StatusCode.UNDEFINED);
             try {
                 retStatus = futureStatus.get();
+                if (retStatus.getCode()
+                        .equals(StatusCode.TIMEOUT)) {
+                    // A timeout happened, lets cleanup the workMonitor
+                    workMonitor.remove(futureStatus.getOrder());
+                }
             } catch (InterruptedException e) {
                 log.error("", e);
             } catch (ExecutionException e) {
@@ -762,6 +782,16 @@ public class ForwardingRulesManager implements
         return true;
     }
 
+    private ConcurrentMap.Entry<Integer, FlowConfig> getStaticFlowEntry(String name, Node node) {
+        for (ConcurrentMap.Entry<Integer, FlowConfig> flowEntry : staticFlows.entrySet()) {
+            FlowConfig flowConfig = flowEntry.getValue();
+            if (flowConfig.isByNameAndNodeIdEqual(name, node)) {
+                return flowEntry;
+            }
+        }
+        return null;
+    }
+
     private void updateLocalDatabase(FlowEntryInstall entry, boolean add) {
         // Update the software view
         updateSwViewes(entry, add);
@@ -1309,7 +1339,6 @@ public class ForwardingRulesManager implements
         retrieveCaches();
     }
 
-    @SuppressWarnings("deprecation")
     private void allocateCaches() {
         if (this.clusterContainerService == null) {
             log.warn("Un-initialized clusterContainerService, can't create cache");
@@ -1353,10 +1382,10 @@ public class ForwardingRulesManager implements
                     EnumSet.of(IClusterServices.cacheMode.TRANSACTIONAL));
 
             clusterContainerService.createCache(WORKSTATUSCACHE,
-                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
+                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
 
             clusterContainerService.createCache(WORKORDERCACHE,
-                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL));
+                    EnumSet.of(IClusterServices.cacheMode.NON_TRANSACTIONAL, IClusterServices.cacheMode.ASYNC));
 
         } catch (CacheConfigException cce) {
             log.error("CacheConfigException");
@@ -1365,7 +1394,7 @@ public class ForwardingRulesManager implements
         }
     }
 
-    @SuppressWarnings({ "unchecked", "deprecation" })
+    @SuppressWarnings({ "unchecked" })
     private void retrieveCaches() {
         ConcurrentMap<?, ?> map;
 
@@ -1814,9 +1843,13 @@ public class ForwardingRulesManager implements
             }
         }
         if (target != null) {
-            // Program the network node
-            Status status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
-                    .installFlowEntry(target.getFlowEntry());
+            Status status = target.validate(container);
+            if (!status.isSuccess()) {
+                log.warn(status.getDescription());
+                return status;
+            }
+            status = (target.installInHw()) ? this.uninstallFlowEntry(target.getFlowEntry()) : this
+                                    .installFlowEntry(target.getFlowEntry());
             if (status.isSuccess()) {
                 // Update Configuration database
                 target.setStatus(SUCCESS);
@@ -1849,27 +1882,46 @@ public class ForwardingRulesManager implements
 
     /**
      * Uninstall all the non-internal Flow Entries present in the software view.
-     * A copy of each entry is stored in the inactive list so that it can be
-     * re-applied when needed. This function is called on the global instance of
-     * FRM only, when the first container is created
+     * If requested, a copy of each original flow entry will be stored in the
+     * inactive list so that it can be re-applied when needed (This is typically
+     * the case when running in the default container and controller moved to
+     * container mode) NOTE WELL: The routine as long as does a bulk change will
+     * operate only on the entries for nodes locally attached so to avoid
+     * redundant operations initiated by multiple nodes
+     *
+     * @param preserveFlowEntries
+     *            if true, a copy of each original entry is stored in the
+     *            inactive list
      */
-    private void uninstallAllFlowEntries() {
+    private void uninstallAllFlowEntries(boolean preserveFlowEntries) {
         log.info("Uninstalling all non-internal flows");
 
+        List<FlowEntryInstall> toRemove = new ArrayList<FlowEntryInstall>();
+
         // Store entries / create target list
         for (ConcurrentMap.Entry<FlowEntryInstall, FlowEntryInstall> mapEntry : installedSwView.entrySet()) {
             FlowEntryInstall flowEntries = mapEntry.getValue();
             // Skip internal generated static flows
             if (!flowEntries.isInternal()) {
-                inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
+                toRemove.add(flowEntries);
+                // Store the original entries if requested
+                if (preserveFlowEntries) {
+                    inactiveFlows.put(flowEntries.getOriginal(), flowEntries.getOriginal());
+                }
             }
         }
 
         // Now remove the entries
-        for (FlowEntry flowEntry : inactiveFlows.keySet()) {
-            Status status = this.removeEntry(flowEntry, false);
-            if (!status.isSuccess()) {
-                log.warn("Failed to remove entry: {}. The failure is: {}", flowEntry, status.getDescription());
+        for (FlowEntryInstall flowEntryHw : toRemove) {
+            Node n = flowEntryHw.getNode();
+            if (n != null && connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
+                Status status = this.removeEntryInternal(flowEntryHw, false);
+                if (!status.isSuccess()) {
+                    log.warn("Failed to remove entry: {}. The failure is: {}", flowEntryHw, status.getDescription());
+                }
+            } else {
+                log.debug("Not removing entry {} because not connected locally, the remote guy will do it's job",
+                        flowEntryHw);
             }
         }
     }
@@ -1912,10 +1964,9 @@ public class ForwardingRulesManager implements
 
     @Override
     public FlowConfig getStaticFlow(String name, Node node) {
-        for (ConcurrentMap.Entry<Integer, FlowConfig> entry : staticFlows.entrySet()) {
-            if (entry.getValue().isByNameAndNodeIdEqual(name, node)) {
-                return entry.getValue();
-            }
+        ConcurrentMap.Entry<Integer, FlowConfig> entry = getStaticFlowEntry(name, node);
+        if(entry != null) {
+            return entry.getValue();
         }
         return null;
     }
@@ -2032,51 +2083,79 @@ public class ForwardingRulesManager implements
         addStaticFlowInternal(allowARP, true); // skip validation on internal static flow name
     }
 
+    /**
+     * (non-Javadoc)
+     *
+     * @see org.opendaylight.controller.switchmanager.ISwitchManagerAware#modeChangeNotify(org.opendaylight.controller.sal.core.Node,
+     *      boolean)
+     *
+     *      This method can be called from within the OSGi framework context,
+     *      given the programming operation can take sometime, it not good
+     *      pratice to have in it's context operations that can take time,
+     *      hence moving off to a different thread for async processing.
+     */
+    private ExecutorService executor;
     @Override
-    public void modeChangeNotify(Node node, boolean proactive) {
-        List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
-
-        List<String> puntAction = new ArrayList<String>();
-        puntAction.add(ActionType.CONTROLLER.toString());
-
-        FlowConfig allowARP = new FlowConfig();
-        allowARP.setInstallInHw(true);
-        allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
-        allowARP.setPriority("1");
-        allowARP.setNode(node);
-        allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue()).toUpperCase());
-        allowARP.setActions(puntAction);
-        defaultConfigs.add(allowARP);
-
-        FlowConfig allowLLDP = new FlowConfig();
-        allowLLDP.setInstallInHw(true);
-        allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
-        allowLLDP.setPriority("1");
-        allowLLDP.setNode(node);
-        allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue()).toUpperCase());
-        allowLLDP.setActions(puntAction);
-        defaultConfigs.add(allowLLDP);
-
-        List<String> dropAction = new ArrayList<String>();
-        dropAction.add(ActionType.DROP.toString());
-
-        FlowConfig dropAllConfig = new FlowConfig();
-        dropAllConfig.setInstallInHw(true);
-        dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop" + FlowConfig.INTERNALSTATICFLOWEND);
-        dropAllConfig.setPriority("0");
-        dropAllConfig.setNode(node);
-        dropAllConfig.setActions(dropAction);
-        defaultConfigs.add(dropAllConfig);
-
-        log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
-        for (FlowConfig fc : defaultConfigs) {
-            Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
-            if (status.isSuccess()) {
-                log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
-            } else {
-                log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"), fc.getName());
+    public void modeChangeNotify(final Node node, final boolean proactive) {
+        Callable<Status> modeChangeCallable = new Callable<Status>() {
+            @Override
+            public Status call() throws Exception {
+                List<FlowConfig> defaultConfigs = new ArrayList<FlowConfig>();
+
+                List<String> puntAction = new ArrayList<String>();
+                puntAction.add(ActionType.CONTROLLER.toString());
+
+                FlowConfig allowARP = new FlowConfig();
+                allowARP.setInstallInHw(true);
+                allowARP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt ARP" + FlowConfig.INTERNALSTATICFLOWEND);
+                allowARP.setPriority("1");
+                allowARP.setNode(node);
+                allowARP.setEtherType("0x" + Integer.toHexString(EtherTypes.ARP.intValue())
+                        .toUpperCase());
+                allowARP.setActions(puntAction);
+                defaultConfigs.add(allowARP);
+
+                FlowConfig allowLLDP = new FlowConfig();
+                allowLLDP.setInstallInHw(true);
+                allowLLDP.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Punt LLDP" + FlowConfig.INTERNALSTATICFLOWEND);
+                allowLLDP.setPriority("1");
+                allowLLDP.setNode(node);
+                allowLLDP.setEtherType("0x" + Integer.toHexString(EtherTypes.LLDP.intValue())
+                        .toUpperCase());
+                allowLLDP.setActions(puntAction);
+                defaultConfigs.add(allowLLDP);
+
+                List<String> dropAction = new ArrayList<String>();
+                dropAction.add(ActionType.DROP.toString());
+
+                FlowConfig dropAllConfig = new FlowConfig();
+                dropAllConfig.setInstallInHw(true);
+                dropAllConfig.setName(FlowConfig.INTERNALSTATICFLOWBEGIN + "Catch-All Drop"
+                        + FlowConfig.INTERNALSTATICFLOWEND);
+                dropAllConfig.setPriority("0");
+                dropAllConfig.setNode(node);
+                dropAllConfig.setActions(dropAction);
+                defaultConfigs.add(dropAllConfig);
+
+                log.info("Forwarding mode for node {} set to {}", node, (proactive ? "proactive" : "reactive"));
+                for (FlowConfig fc : defaultConfigs) {
+                    Status status = (proactive) ? addStaticFlowInternal(fc, false) : removeStaticFlow(fc);
+                    if (status.isSuccess()) {
+                        log.info("{} Proactive Static flow: {}", (proactive ? "Installed" : "Removed"), fc.getName());
+                    } else {
+                        log.warn("Failed to {} Proactive Static flow: {}", (proactive ? "install" : "remove"),
+                                fc.getName());
+                    }
+                }
+                return new Status(StatusCode.SUCCESS);
             }
-        }
+        };
+
+        /*
+         * Execute the work outside the caller context, this could be an
+         * expensive operation and we don't want to block the caller for it.
+         */
+        this.executor.submit(modeChangeCallable);
     }
 
     /**
@@ -2128,7 +2207,99 @@ public class ForwardingRulesManager implements
 
     @Override
     public void notifyNodeConnector(NodeConnector nodeConnector, UpdateType type, Map<String, Property> propMap) {
+        boolean updateStaticFlowCluster = false;
 
+        switch (type) {
+        case ADDED:
+            break;
+        case CHANGED:
+            Config config = (propMap == null) ? null : (Config) propMap.get(Config.ConfigPropName);
+            if (config != null) {
+                switch (config.getValue()) {
+                case Config.ADMIN_DOWN:
+                    log.trace("Port {} is administratively down: uninstalling interested flows", nodeConnector);
+                    updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
+                    break;
+                case Config.ADMIN_UP:
+                    log.trace("Port {} is administratively up: installing interested flows", nodeConnector);
+                    updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nodeConnector);
+                    break;
+                case Config.ADMIN_UNDEF:
+                    break;
+                default:
+                }
+            }
+            break;
+        case REMOVED:
+            // This is the case where a switch port is removed from the SDN agent space
+            log.trace("Port {} was removed from our control: uninstalling interested flows", nodeConnector);
+            updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nodeConnector);
+            break;
+        default:
+
+        }
+
+        if (updateStaticFlowCluster) {
+            refreshClusterStaticFlowsStatus(nodeConnector.getNode());
+        }
+    }
+
+    /*
+     * It goes through the static flows configuration, it identifies the ones
+     * which have the specified node connector as input or output port and
+     * install them on the network node if they are marked to be installed in
+     * hardware and their status shows they were not installed yet
+     */
+    private boolean installFlowsOnNodeConnectorUp(NodeConnector nodeConnector) {
+        boolean updated = false;
+        List<FlowConfig> flowConfigForNode = getStaticFlows(nodeConnector.getNode());
+        for (FlowConfig flowConfig : flowConfigForNode) {
+            if (doesFlowContainNodeConnector(flowConfig.getFlow(), nodeConnector)) {
+                if (flowConfig.installInHw() && !flowConfig.getStatus().equals(SUCCESS)) {
+                    Status status = this.installFlowEntry(flowConfig.getFlowEntry());
+                    if (!status.isSuccess()) {
+                        flowConfig.setStatus(status.getDescription());
+                    } else {
+                        flowConfig.setStatus(SUCCESS);
+                    }
+                    updated = true;
+                }
+            }
+        }
+        return updated;
+    }
+
+    /*
+     * Remove from the network node all the flows which have the specified node
+     * connector as input or output port. If any of the flow entry is a static
+     * flow, it updates the correspondent configuration.
+     */
+    private boolean removeFlowsOnNodeConnectorDown(NodeConnector nodeConnector) {
+        boolean updated = false;
+        List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nodeConnector.getNode());
+        if (nodeFlowEntries == null) {
+            return updated;
+        }
+        for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
+            if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nodeConnector)) {
+                Status status = this.removeEntryInternal(fei, true);
+                if (!status.isSuccess()) {
+                    continue;
+                }
+                /*
+                 * If the flow entry is a static flow, then update its
+                 * configuration
+                 */
+                if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
+                    FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
+                    if (flowConfig != null) {
+                        flowConfig.setStatus(PORTREMOVED);
+                        updated = true;
+                    }
+                }
+            }
+        }
+        return updated;
     }
 
     private FlowConfig getDerivedFlowConfig(FlowConfig original, String configName, Short port) {
@@ -2416,6 +2587,14 @@ public class ForwardingRulesManager implements
                             } else {
                                 log.warn("Not expected null WorkStatus", work);
                             }
+                        }  else if (event instanceof ContainerFlowChangeEvent) {
+                            /*
+                             * Whether it is an addition or removal, we have to
+                             * recompute the merged flows entries taking into
+                             * account all the current container flows because
+                             * flow merging is not an injective function
+                             */
+                            updateFlowsContainerFlow();
                         } else {
                             log.warn("Dequeued unknown event {}", event.getClass()
                                     .getSimpleName());
@@ -2453,6 +2632,9 @@ public class ForwardingRulesManager implements
         // Initialize graceful stop flag
         stopping = false;
 
+        // Allocate the executor service
+        this.executor = Executors.newSingleThreadExecutor();
+
         // Start event handler thread
         frmEventHandler.start();
 
@@ -2472,7 +2654,15 @@ public class ForwardingRulesManager implements
      */
     void stop() {
         stopping = true;
-        uninstallAllFlowEntries();
+        uninstallAllFlowEntries(false);
+        // Shutdown executor
+        this.executor.shutdownNow();
+        // Now walk all the workMonitor and wake up the one sleeping because
+        // destruction is happening
+        for (FlowEntryDistributionOrder fe : workMonitor.keySet()) {
+            FlowEntryDistributionOrderFutureTask task = workMonitor.get(fe);
+            task.cancel(true);
+        }
     }
 
     public void setFlowProgrammerService(IFlowProgrammerService service) {
@@ -2509,12 +2699,8 @@ public class ForwardingRulesManager implements
         }
         log.trace("Container {}: Updating installed flows because of container flow change: {} {}",
                 container.getName(), t, current);
-        /*
-         * Whether it is an addition or removal, we have to recompute the merged
-         * flows entries taking into account all the current container flows
-         * because flow merging is not an injective function
-         */
-        updateFlowsContainerFlow();
+        ContainerFlowChangeEvent ev = new ContainerFlowChangeEvent(previous, current, t);
+        pendingEvents.offer(ev);
     }
 
     @Override
@@ -2527,57 +2713,22 @@ public class ForwardingRulesManager implements
 
         switch (t) {
         case REMOVED:
-
-            List<FlowEntryInstall> nodeFlowEntries = nodeFlows.get(nc.getNode());
-            if (nodeFlowEntries == null) {
-                return;
-            }
-            for (FlowEntryInstall fei : new ArrayList<FlowEntryInstall>(nodeFlowEntries)) {
-                if (doesFlowContainNodeConnector(fei.getInstall().getFlow(), nc)) {
-                    Status status = this.removeEntryInternal(fei, true);
-                    if (!status.isSuccess()) {
-                        continue;
-                    }
-                    /*
-                     * If the flow entry is a static flow, then update its
-                     * configuration
-                     */
-                    if (fei.getGroupName().equals(FlowConfig.STATICFLOWGROUP)) {
-                        FlowConfig flowConfig = getStaticFlow(fei.getFlowName(), fei.getNode());
-                        if (flowConfig != null) {
-                            flowConfig.setStatus(PORTREMOVED);
-                            updateStaticFlowCluster = true;
-                        }
-                    }
-                }
-            }
-            if (updateStaticFlowCluster) {
-                refreshClusterStaticFlowsStatus(nc.getNode());
-            }
+            log.trace("Port {} was removed from container: uninstalling interested flows", nc);
+            updateStaticFlowCluster = removeFlowsOnNodeConnectorDown(nc);
             break;
         case ADDED:
-            List<FlowConfig> flowConfigForNode = getStaticFlows(nc.getNode());
-            for (FlowConfig flowConfig : flowConfigForNode) {
-                if (doesFlowContainNodeConnector(flowConfig.getFlow(), nc)) {
-                    if (flowConfig.installInHw()) {
-                        Status status = this.installFlowEntry(flowConfig.getFlowEntry());
-                        if (!status.isSuccess()) {
-                            flowConfig.setStatus(status.getDescription());
-                        } else {
-                            flowConfig.setStatus(SUCCESS);
-                        }
-                        updateStaticFlowCluster = true;
-                    }
-                }
-            }
-            if (updateStaticFlowCluster) {
-                refreshClusterStaticFlowsStatus(nc.getNode());
-            }
+            log.trace("Port {} was added to container: reinstall interested flows", nc);
+            updateStaticFlowCluster = installFlowsOnNodeConnectorUp(nc);
+
             break;
         case CHANGED:
             break;
         default:
         }
+
+        if (updateStaticFlowCluster) {
+            refreshClusterStaticFlowsStatus(nc.getNode());
+        }
     }
 
     @Override
@@ -2588,8 +2739,15 @@ public class ForwardingRulesManager implements
         }
         switch (update) {
         case ADDED:
+            /*
+             * Controller is moving to container mode. We are in the default
+             * container context, we need to remove all our non-internal flows
+             * to prevent any container isolation breakage. We also need to
+             * preserve our flow so that they can be re-installed if we move
+             * back to non container mode (no containers).
+             */
             this.inContainerMode = true;
-            this.uninstallAllFlowEntries();
+            this.uninstallAllFlowEntries(true);
             break;
         case REMOVED:
             this.inContainerMode = false;
@@ -2675,6 +2833,30 @@ public class ForwardingRulesManager implements
             return newEntry;
         }
     }
+    private class ContainerFlowChangeEvent extends FRMEvent {
+        private final ContainerFlow previous;
+        private final ContainerFlow current;
+        private final UpdateType type;
+
+        public ContainerFlowChangeEvent(ContainerFlow previous, ContainerFlow current, UpdateType type) {
+            this.previous = previous;
+            this.current = current;
+            this.type = type;
+        }
+
+        public ContainerFlow getPrevious() {
+            return this.previous;
+        }
+
+        public ContainerFlow getCurrent() {
+            return this.current;
+        }
+
+        public UpdateType getType() {
+            return this.type;
+        }
+    }
+
 
     private class WorkStatusCleanup extends FRMEvent {
         private FlowEntryDistributionOrder fe;
@@ -2872,6 +3054,36 @@ public class ForwardingRulesManager implements
         }
     }
 
+    public void _frmProcessErrorEvent(CommandInterpreter ci) throws UnknownHostException {
+        Node node = null;
+        long reqId = 0L;
+        String nodeId = ci.nextArgument();
+        if (nodeId == null) {
+            ci.print("Node id not specified");
+            return;
+        }
+        String requestId = ci.nextArgument();
+        if (requestId == null) {
+            ci.print("Request id not specified");
+            return;
+        }
+        try {
+            node = NodeCreator.createOFNode(Long.valueOf(nodeId));
+        } catch (NumberFormatException e) {
+            ci.print("Node id not a number");
+            return;
+        }
+        try {
+            reqId = Long.parseLong(requestId);
+        } catch (NumberFormatException e) {
+            ci.print("Request id not a number");
+            return;
+        }
+        // null for error object is good enough for now
+        ErrorReportedEvent event = new ErrorReportedEvent(reqId, node, null);
+        this.processErrorEvent(event);
+    }
+
     @Override
     public void flowRemoved(Node node, Flow flow) {
         log.trace("Received flow removed notification on {} for {}", node, flow);
@@ -2924,16 +3136,32 @@ public class ForwardingRulesManager implements
          * mapping will have to be added in future
          */
         FlowEntryInstall target = null;
-        for (FlowEntryInstall index : nodeFlows.get(node)) {
-            FlowEntryInstall entry = installedSwView.get(index);
-            if (entry.getRequestId() == rid) {
-                target = entry;
-                break;
+        List<FlowEntryInstall> flowEntryInstallList = nodeFlows.get(node);
+        // flowEntryInstallList could be null.
+        // so check for it.
+        if(flowEntryInstallList != null) {
+            for (FlowEntryInstall index : flowEntryInstallList) {
+                FlowEntryInstall entry = installedSwView.get(index);
+                if(entry != null) {
+                    if (entry.getRequestId() == rid) {
+                        target = entry;
+                        break;
+                    }
+                }
             }
         }
         if (target != null) {
             // This was a flow install, update database
             this.updateLocalDatabase(target, false);
+            // also update the config
+            if(FlowConfig.STATICFLOWGROUP.equals(target.getGroupName())) {
+                ConcurrentMap.Entry<Integer, FlowConfig> staticFlowEntry = getStaticFlowEntry(target.getFlowName(),target.getNode());
+                // staticFlowEntry should never be null.
+                // the null check is just an extra defensive check.
+                if(staticFlowEntry != null) {
+                    staticFlows.remove(staticFlowEntry.getKey());
+                }
+            }
         }
 
         // Notify listeners
@@ -3002,7 +3230,7 @@ public class ForwardingRulesManager implements
                 return;
             }
             Node n = fei.getNode();
-            if (connectionManager.isLocal(n)) {
+            if (connectionManager.getLocalityStatus(n) == ConnectionLocality.LOCAL) {
                 logsync.trace("workOrder for fe {} processed locally", fe);
                 // I'm the controller in charge for the request, queue it for
                 // processing
@@ -3021,7 +3249,7 @@ public class ForwardingRulesManager implements
              */
             if (fe.getRequestorController()
                     .equals(clusterContainerService.getMyAddress())) {
-                FlowEntryDistributionOrderFutureTask fet = workMonitor.get(fe);
+                FlowEntryDistributionOrderFutureTask fet = workMonitor.remove(fe);
                 if (fet != null) {
                     logsync.trace("workStatus response is for us {}", fe);
                     // Signal we got the status