Controller ignores switch, if no ports are present
[controller.git] / opendaylight / protocol_plugins / openflow / src / main / java / org / opendaylight / controller / protocol_plugin / openflow / internal / InventoryServiceShim.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.protocol_plugin.openflow.internal;
10
11 import java.util.ArrayList;
12 import java.util.Collections;
13 import java.util.Date;
14 import java.util.HashSet;
15 import java.util.List;
16 import java.util.Map;
17 import java.util.Set;
18 import java.util.concurrent.ConcurrentHashMap;
19 import java.util.concurrent.ConcurrentMap;
20 import java.util.concurrent.CopyOnWriteArrayList;
21 import java.util.concurrent.CopyOnWriteArraySet;
22
23 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimExternalListener;
24 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimInternalListener;
25 import org.opendaylight.controller.protocol_plugin.openflow.IOFStatisticsListener;
26 import org.opendaylight.controller.protocol_plugin.openflow.core.IController;
27 import org.opendaylight.controller.protocol_plugin.openflow.core.IMessageListener;
28 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitch;
29 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitchStateListener;
30 import org.opendaylight.controller.sal.action.SupportedFlowActions;
31 import org.opendaylight.controller.sal.connection.ConnectionLocality;
32 import org.opendaylight.controller.sal.connection.IPluginOutConnectionService;
33 import org.opendaylight.controller.sal.core.Buffers;
34 import org.opendaylight.controller.sal.core.Capabilities;
35 import org.opendaylight.controller.sal.core.ContainerFlow;
36 import org.opendaylight.controller.sal.core.Description;
37 import org.opendaylight.controller.sal.core.IContainerAware;
38 import org.opendaylight.controller.sal.core.IContainerListener;
39 import org.opendaylight.controller.sal.core.MacAddress;
40 import org.opendaylight.controller.sal.core.Node;
41 import org.opendaylight.controller.sal.core.NodeConnector;
42 import org.opendaylight.controller.sal.core.Property;
43 import org.opendaylight.controller.sal.core.Tables;
44 import org.opendaylight.controller.sal.core.TimeStamp;
45 import org.opendaylight.controller.sal.core.UpdateType;
46 import org.opendaylight.controller.sal.utils.GlobalConstants;
47 import org.opendaylight.controller.sal.utils.NodeCreator;
48 import org.openflow.protocol.OFMessage;
49 import org.openflow.protocol.OFPortStatus;
50 import org.openflow.protocol.OFPortStatus.OFPortReason;
51 import org.openflow.protocol.OFType;
52 import org.openflow.protocol.statistics.OFDescriptionStatistics;
53 import org.openflow.protocol.statistics.OFStatistics;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56
57 /**
58  * The class describes a shim layer that bridges inventory events from Openflow
59  * core to various listeners. The notifications are filtered based on container
60  * configurations.
61  *
62  *
63  */
64 public class InventoryServiceShim implements IContainerListener,
65         IMessageListener, ISwitchStateListener, IOFStatisticsListener, IContainerAware {
66     protected static final Logger logger = LoggerFactory
67             .getLogger(InventoryServiceShim.class);
68     private IController controller = null;
69     private final ConcurrentMap<String, IInventoryShimInternalListener> inventoryShimInternalListeners = new ConcurrentHashMap<String, IInventoryShimInternalListener>();
70     private final Set<IInventoryShimInternalListener> globalInventoryShimInternalListeners = new HashSet<IInventoryShimInternalListener>();
71     private final List<IInventoryShimExternalListener> inventoryShimExternalListeners = new CopyOnWriteArrayList<IInventoryShimExternalListener>();
72     private final ConcurrentMap<NodeConnector, Set<String>> nodeConnectorContainerMap = new ConcurrentHashMap<NodeConnector, Set<String>>();
73     private final ConcurrentMap<Node, Set<String>> nodeContainerMap = new ConcurrentHashMap<Node, Set<String>>();
74     private final ConcurrentMap<NodeConnector, Set<Property>> nodeConnectorProps = new ConcurrentHashMap<NodeConnector, Set<Property>>();
75     private final ConcurrentMap<Node, Set<Property>> nodeProps = new ConcurrentHashMap<Node, Set<Property>>();
76     private IPluginOutConnectionService connectionOutService;
77
78     void setController(IController s) {
79         this.controller = s;
80     }
81
82     void unsetController(IController s) {
83         if (this.controller == s) {
84             this.controller = null;
85         }
86     }
87
88     void setInventoryShimGlobalInternalListener(Map<?, ?> props,
89             IInventoryShimInternalListener s) {
90         if ((this.globalInventoryShimInternalListeners != null)) {
91             this.globalInventoryShimInternalListeners.add(s);
92         }
93     }
94
95     void unsetInventoryShimGlobalInternalListener(Map<?, ?> props,
96             IInventoryShimInternalListener s) {
97         if ((this.globalInventoryShimInternalListeners != null)) {
98             this.globalInventoryShimInternalListeners.remove(s);
99         }
100     }
101
102     void setInventoryShimInternalListener(Map<?, ?> props,
103             IInventoryShimInternalListener s) {
104         if (props == null) {
105             logger.error("setInventoryShimInternalListener property is null");
106             return;
107         }
108         String containerName = (String) props.get("containerName");
109         if (containerName == null) {
110             logger.error("setInventoryShimInternalListener containerName not supplied");
111             return;
112         }
113         if ((this.inventoryShimInternalListeners != null)
114                 && !this.inventoryShimInternalListeners.containsValue(s)) {
115             this.inventoryShimInternalListeners.put(containerName, s);
116             logger.trace(
117                     "Added inventoryShimInternalListener for container {}",
118                     containerName);
119         }
120     }
121
122     void unsetInventoryShimInternalListener(Map<?, ?> props,
123             IInventoryShimInternalListener s) {
124         if (props == null) {
125             logger.error("unsetInventoryShimInternalListener property is null");
126             return;
127         }
128         String containerName = (String) props.get("containerName");
129         if (containerName == null) {
130             logger.error("setInventoryShimInternalListener containerName not supplied");
131             return;
132         }
133         if ((this.inventoryShimInternalListeners != null)
134                 && this.inventoryShimInternalListeners.get(containerName) != null
135                 && this.inventoryShimInternalListeners.get(containerName)
136                         .equals(s)) {
137             this.inventoryShimInternalListeners.remove(containerName);
138             logger.trace(
139                     "Removed inventoryShimInternalListener for container {}",
140                     containerName);
141         }
142     }
143
144     void setInventoryShimExternalListener(IInventoryShimExternalListener s) {
145         logger.trace("Set inventoryShimExternalListener {}", s);
146         if ((this.inventoryShimExternalListeners != null)
147                 && !this.inventoryShimExternalListeners.contains(s)) {
148             this.inventoryShimExternalListeners.add(s);
149         }
150     }
151
152     void unsetInventoryShimExternalListener(IInventoryShimExternalListener s) {
153         logger.trace("Unset inventoryShimExternalListener {}", s);
154         if ((this.inventoryShimExternalListeners != null)
155                 && this.inventoryShimExternalListeners.contains(s)) {
156             this.inventoryShimExternalListeners.remove(s);
157         }
158     }
159
160     void setIPluginOutConnectionService(IPluginOutConnectionService s) {
161         connectionOutService = s;
162     }
163
164     void unsetIPluginOutConnectionService(IPluginOutConnectionService s) {
165         if (connectionOutService == s) {
166             connectionOutService = null;
167         }
168     }
169
170     /**
171      * Function called by the dependency manager when all the required
172      * dependencies are satisfied
173      *
174      */
175     void init() {
176         this.controller.addMessageListener(OFType.PORT_STATUS, this);
177         this.controller.addSwitchStateListener(this);
178     }
179
180     /**
181      * Function called after registering the service in OSGi service registry.
182      */
183     void started() {
184         /* Start with existing switches */
185         startService();
186     }
187
188     /**
189      * Function called by the dependency manager when at least one dependency
190      * become unsatisfied or when the component is shutting down because for
191      * example bundle is being stopped.
192      *
193      */
194     void destroy() {
195         this.controller.removeMessageListener(OFType.PACKET_IN, this);
196         this.controller.removeSwitchStateListener(this);
197
198         this.inventoryShimInternalListeners.clear();
199         this.nodeConnectorContainerMap.clear();
200         this.nodeContainerMap.clear();
201         this.globalInventoryShimInternalListeners.clear();
202         this.controller = null;
203     }
204
205     @Override
206     public void receive(ISwitch sw, OFMessage msg) {
207         if (msg instanceof OFPortStatus) {
208             handlePortStatusMessage(sw, (OFPortStatus) msg);
209         }
210         return;
211     }
212
213     protected void handlePortStatusMessage(ISwitch sw, OFPortStatus m) {
214         Node node = NodeCreator.createOFNode(sw.getId());
215         NodeConnector nodeConnector = PortConverter.toNodeConnector(
216             m.getDesc().getPortNumber(), node);
217         // get node connector properties
218         Set<Property> props = InventoryServiceHelper.OFPortToProps(m.getDesc());
219
220         UpdateType type = null;
221         if (m.getReason() == (byte) OFPortReason.OFPPR_ADD.ordinal()) {
222             type = UpdateType.ADDED;
223             nodeConnectorProps.put(nodeConnector, props);
224         } else if (m.getReason() == (byte) OFPortReason.OFPPR_DELETE.ordinal()) {
225             type = UpdateType.REMOVED;
226             nodeConnectorProps.remove(nodeConnector);
227         } else if (m.getReason() == (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
228             type = UpdateType.CHANGED;
229             nodeConnectorProps.put(nodeConnector, props);
230         }
231
232         logger.trace("handlePortStatusMessage {} type {}", nodeConnector, type);
233
234         if (type != null) {
235             notifyInventoryShimListener(nodeConnector, type, props);
236         }
237     }
238
239     @Override
240     public void switchAdded(ISwitch sw) {
241         if (sw == null) {
242             logger.debug("Ignore null switch addition");
243             return;
244         }
245         Node node = NodeCreator.createOFNode(sw.getId());
246         if ((nodeProps.get(node) != null)  && (connectionOutService.isLocal(node))) {
247             logger.debug("Ignore switchAdded {}", sw);
248             return;
249         }
250
251         // Add all the nodeConnectors of this switch if any
252         Map<NodeConnector, Set<Property>> ncProps = InventoryServiceHelper.OFSwitchToProps(sw);
253         if (!ncProps.isEmpty()) {
254             for (Map.Entry<NodeConnector, Set<Property>> entry : ncProps.entrySet()) {
255                 Set<Property> props = new HashSet<Property>();
256                 Set<Property> prop = entry.getValue();
257                 if (prop != null) {
258                     props.addAll(prop);
259                 }
260                 nodeConnectorProps.put(entry.getKey(), props);
261                 notifyInventoryShimListener(entry.getKey(), UpdateType.ADDED, entry.getValue());
262             }
263         } else {
264             /*
265              * If no node connector is present, publish the node addition itself
266              * in order to let Connection Manager properly set the node locality
267              */
268             this.notifyInventoryShimListener(node, UpdateType.ADDED, Collections.<Property>emptySet());
269         }
270
271         // Add this node
272         if (connectionOutService.getLocalityStatus(node) != ConnectionLocality.NOT_CONNECTED) {
273             addNode(sw);
274         } else {
275             logger.debug("Skipping node addition due to Connectivity Status : {}", connectionOutService.getLocalityStatus(node).name());
276         }
277     }
278
279     @Override
280     public void switchDeleted(ISwitch sw) {
281         if (sw == null) {
282             return;
283         }
284
285         removeNode(sw);
286     }
287
288     @Override
289     public void containerModeUpdated(UpdateType t) {
290         // do nothing
291     }
292
293     @Override
294     public void tagUpdated(String containerName, Node n, short oldTag,
295             short newTag, UpdateType t) {
296         logger.debug("tagUpdated: {} type {} for container {}", new Object[] {
297                 n, t, containerName });
298     }
299
300     @Override
301     public void containerFlowUpdated(String containerName,
302             ContainerFlow previousFlow, ContainerFlow currentFlow, UpdateType t) {
303     }
304
305     @Override
306     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
307         logger.debug("nodeConnectorUpdated: {} type {} for container {}", new Object[] { nc, t, containerName });
308         Node node = nc.getNode();
309         Set<String> ncContainers = this.nodeConnectorContainerMap.get(nc);
310         Set<String> nodeContainers = this.nodeContainerMap.get(node);
311         if (ncContainers == null) {
312             ncContainers = new CopyOnWriteArraySet<String>();
313         }
314         if (nodeContainers == null) {
315             nodeContainers = new CopyOnWriteArraySet<String>();
316         }
317         boolean notifyNodeUpdate = false;
318
319         switch (t) {
320         case ADDED:
321             if (ncContainers.add(containerName)) {
322                 this.nodeConnectorContainerMap.put(nc, ncContainers);
323             }
324             if (nodeContainers.add(containerName)) {
325                 this.nodeContainerMap.put(node, nodeContainers);
326                 notifyNodeUpdate = true;
327             }
328             break;
329         case REMOVED:
330             if (ncContainers.remove(containerName)) {
331                 if (ncContainers.isEmpty()) {
332                     // Do cleanup to reduce memory footprint if no
333                     // elements to be tracked
334                     this.nodeConnectorContainerMap.remove(nc);
335                 } else {
336                     this.nodeConnectorContainerMap.put(nc, ncContainers);
337                 }
338             }
339             boolean nodeContainerUpdate = true;
340             for (NodeConnector ncContainer : nodeConnectorContainerMap.keySet()) {
341                 if ((ncContainer.getNode().equals(node)) && (nodeConnectorContainerMap.get(ncContainer).contains(containerName))) {
342                     nodeContainerUpdate = false;
343                     break;
344                 }
345             }
346             if (nodeContainerUpdate) {
347                 nodeContainers.remove(containerName);
348                 notifyNodeUpdate = true;
349                 if (nodeContainers.isEmpty()) {
350                     this.nodeContainerMap.remove(node);
351                 } else {
352                     this.nodeContainerMap.put(node, nodeContainers);
353                 }
354             }
355             break;
356         case CHANGED:
357             break;
358         }
359
360         Set<Property> nodeProp = nodeProps.get(node);
361         if (nodeProp == null) {
362             return;
363         }
364         Set<Property> ncProp = nodeConnectorProps.get(nc);
365         // notify InventoryService
366         notifyInventoryShimInternalListener(containerName, nc, t, ncProp);
367
368         if (notifyNodeUpdate) {
369             notifyInventoryShimInternalListener(containerName, node, t, nodeProp);
370         }
371     }
372
373     private void notifyInventoryShimExternalListener(Node node, UpdateType type, Set<Property> props) {
374         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
375             s.updateNode(node, type, props);
376         }
377     }
378
379     private void notifyInventoryShimExternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
380         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
381             s.updateNodeConnector(nodeConnector, type, props);
382         }
383     }
384
385     private void notifyInventoryShimInternalListener(String container,
386             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
387         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners.get(container);
388         if (inventoryShimInternalListener != null) {
389             inventoryShimInternalListener.updateNodeConnector(nodeConnector, type, props);
390             logger.trace("notifyInventoryShimInternalListener {} type {} for container {}", new Object[] {
391                     nodeConnector, type, container });
392         }
393     }
394
395     /*
396      * Notify all internal and external listeners
397      */
398     private void notifyInventoryShimListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
399
400         //establish locality before notifying
401         boolean isNodeLocal;
402         if (type == UpdateType.REMOVED){
403             //if removing get the locality first
404             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
405             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
406         } else {
407             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
408             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
409         }
410
411         if (isNodeLocal) {
412             // notify other containers
413             Set<String> containers = (nodeConnectorContainerMap.get(nodeConnector) == null) ? new HashSet<String>()
414                     : new HashSet<String>(nodeConnectorContainerMap.get(nodeConnector));
415             containers.add(GlobalConstants.DEFAULT.toString());
416             for (String container : containers) {
417                 notifyInventoryShimInternalListener(container, nodeConnector, type, props);
418             }
419
420             // Notify plugin listeners (Discovery, DataPacket, OFstats etc.)
421             notifyInventoryShimExternalListener(nodeConnector, type, props);
422
423             logger.debug("Connection service accepted the inventory notification for {} {}", nodeConnector, type);
424         } else {
425             logger.debug("Connection service dropped the inventory notification for {} {}", nodeConnector, type);
426         }
427     }
428
429     /*
430      * Notify all internal and external listeners
431      */
432     private void notifyInventoryShimListener(Node node, UpdateType type, Set<Property> props) {
433
434         //establish locality before notifying
435         boolean isNodeLocal;
436         if (type == UpdateType.REMOVED){
437             //if removing get the locality first
438             isNodeLocal = connectionOutService.isLocal(node);
439             notifyGlobalInventoryShimInternalListener(node, type, props);
440         } else {
441             notifyGlobalInventoryShimInternalListener(node, type, props);
442             isNodeLocal = connectionOutService.isLocal(node);
443         }
444
445         if (isNodeLocal) {
446             // Now notify other containers
447             Set<String> containers = (nodeContainerMap.get(node) == null) ? new HashSet<String>()
448                     : new HashSet<String>(nodeContainerMap.get(node));
449             containers.add(GlobalConstants.DEFAULT.toString());
450             for (String container : containers) {
451                 notifyInventoryShimInternalListener(container, node, type, props);
452             }
453
454             // Notify plugin listeners (Discovery, DataPacket, OFstats etc.)
455             notifyInventoryShimExternalListener(node, type, props);
456
457             logger.debug("Connection service accepted the inventory notification for {} {}", node, type);
458         } else {
459             logger.debug("Connection service dropped the inventory notification for {} {}", node, type);
460         }
461     }
462
463     private void notifyGlobalInventoryShimInternalListener(Node node, UpdateType type, Set<Property> props) {
464         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
465             globalListener.updateNode(node, type, props);
466             logger.trace("notifyGlobalInventoryShimInternalListener {} type {}", new Object[] { node, type });
467         }
468     }
469
470     private void notifyGlobalInventoryShimInternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
471         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
472             globalListener.updateNodeConnector(nodeConnector, type, props);
473             logger.trace(
474                     "notifyGlobalInventoryShimInternalListener {} type {}",
475                     new Object[] { nodeConnector, type });
476         }
477     }
478
479     private void notifyInventoryShimInternalListener(String container,
480             Node node, UpdateType type, Set<Property> props) {
481         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
482                 .get(container);
483         if (inventoryShimInternalListener != null) {
484             inventoryShimInternalListener.updateNode(node, type, props);
485             logger.trace(
486                     "notifyInventoryShimInternalListener {} type {} for container {}",
487                     new Object[] { node, type, container });
488         }
489     }
490
491     private void addNode(ISwitch sw) {
492         Node node = NodeCreator.createOFNode(sw.getId());
493         UpdateType type = UpdateType.ADDED;
494
495         Set<Property> props = new HashSet<Property>();
496         Long sid = (Long) node.getID();
497
498         Date connectedSince = sw.getConnectedDate();
499         Long connectedSinceTime = (connectedSince == null) ? 0 : connectedSince
500                 .getTime();
501         props.add(new TimeStamp(connectedSinceTime, "connectedSince"));
502         props.add(new MacAddress(deriveMacAddress(sid)));
503
504         byte tables = sw.getTables();
505         Tables t = new Tables(tables);
506         if (t != null) {
507             props.add(t);
508         }
509         int cap = sw.getCapabilities();
510         Capabilities c = new Capabilities(cap);
511         if (c != null) {
512             props.add(c);
513         }
514         int act = sw.getActions();
515         SupportedFlowActions a = new SupportedFlowActions(FlowConverter.getFlowActions(act));
516         if (a != null) {
517             props.add(a);
518         }
519         int buffers = sw.getBuffers();
520         Buffers b = new Buffers(buffers);
521         if (b != null) {
522             props.add(b);
523         }
524
525         if ((nodeProps.get(node) == null) &&  (connectionOutService.isLocal(node)))  {
526             // The switch is connected for the first time, flush all flows
527             // that may exist on this switch
528             sw.deleteAllFlows();
529        }
530         nodeProps.put(node, props);
531         // Notify all internal and external listeners
532         notifyInventoryShimListener(node, type, props);
533     }
534
535     private void removeNode(ISwitch sw) {
536         Node node = NodeCreator.createOFNode(sw.getId());
537         if(node == null) {
538             return;
539         }
540         removeNodeConnectorProps(node);
541         nodeProps.remove(node);
542         UpdateType type = UpdateType.REMOVED;
543         // Notify all internal and external listeners
544         notifyInventoryShimListener(node, type, null);
545     }
546
547     private void startService() {
548         // Get a snapshot of all the existing switches
549         Map<Long, ISwitch> switches = this.controller.getSwitches();
550         for (ISwitch sw : switches.values()) {
551             switchAdded(sw);
552         }
553     }
554
555     private void removeNodeConnectorProps(Node node) {
556         List<NodeConnector> ncList = new ArrayList<NodeConnector>();
557         for (NodeConnector nc : nodeConnectorProps.keySet()) {
558             if (nc.getNode().equals(node)) {
559                 ncList.add(nc);
560             }
561         }
562         for (NodeConnector nc : ncList) {
563             nodeConnectorProps.remove(nc);
564         }
565     }
566
567     @Override
568     public void descriptionStatisticsRefreshed(Long switchId, List<OFStatistics> descriptionStats) {
569         Node node = NodeCreator.createOFNode(switchId);
570         Set<Property> properties = new HashSet<Property>(1);
571         OFDescriptionStatistics ofDesc = (OFDescriptionStatistics) descriptionStats.get(0);
572         Description desc = new Description(ofDesc.getDatapathDescription());
573         properties.add(desc);
574
575         // Notify all internal and external listeners
576         notifyInventoryShimListener(node, UpdateType.CHANGED, properties);
577     }
578
579     private byte[] deriveMacAddress(long dpid) {
580         byte[] mac = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
581
582         for (short i = 0; i < 6; i++) {
583             mac[5 - i] = (byte) dpid;
584             dpid >>= 8;
585         }
586
587         return mac;
588     }
589
590     @Override
591     public void flowStatisticsRefreshed(Long switchId, List<OFStatistics> flows) {
592         // Nothing to do
593     }
594
595     @Override
596     public void portStatisticsRefreshed(Long switchId, List<OFStatistics> ports) {
597         // Nothing to do
598     }
599
600     @Override
601     public void tableStatisticsRefreshed(Long switchId, List<OFStatistics> tables) {
602         // Nothing to do
603     }
604
605     @Override
606     public void containerCreate(String containerName) {
607         // Nothing to do
608     }
609
610     @Override
611     public void containerDestroy(String containerName) {
612         Set<NodeConnector> removeNodeConnectorSet = new HashSet<NodeConnector>();
613         Set<Node> removeNodeSet = new HashSet<Node>();
614         for (Map.Entry<NodeConnector, Set<String>> entry : nodeConnectorContainerMap.entrySet()) {
615             Set<String> ncContainers = entry.getValue();
616             if (ncContainers.contains(containerName)) {
617                 NodeConnector nodeConnector = entry.getKey();
618                 removeNodeConnectorSet.add(nodeConnector);
619             }
620         }
621         for (Map.Entry<Node, Set<String>> entry : nodeContainerMap.entrySet()) {
622             Set<String> nodeContainers = entry.getValue();
623             if (nodeContainers.contains(containerName)) {
624                 Node node = entry.getKey();
625                 removeNodeSet.add(node);
626             }
627         }
628         for (NodeConnector nodeConnector : removeNodeConnectorSet) {
629             Set<String> ncContainers = nodeConnectorContainerMap.get(nodeConnector);
630             ncContainers.remove(containerName);
631             if (ncContainers.isEmpty()) {
632                 nodeConnectorContainerMap.remove(nodeConnector);
633             }
634         }
635         for (Node node : removeNodeSet) {
636             Set<String> nodeContainers = nodeContainerMap.get(node);
637             nodeContainers.remove(containerName);
638             if (nodeContainers.isEmpty()) {
639                 nodeContainerMap.remove(node);
640             }
641         }
642     }
643 }