6401ec52f81faba1694302314882affb0d818ca5
[controller.git] / opendaylight / protocol_plugins / openflow / src / main / java / org / opendaylight / controller / protocol_plugin / openflow / internal / InventoryServiceShim.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.protocol_plugin.openflow.internal;
10
11 import java.util.ArrayList;
12 import java.util.Date;
13 import java.util.HashSet;
14 import java.util.List;
15 import java.util.Map;
16 import java.util.Set;
17 import java.util.concurrent.ConcurrentHashMap;
18 import java.util.concurrent.ConcurrentMap;
19 import java.util.concurrent.CopyOnWriteArrayList;
20 import java.util.concurrent.CopyOnWriteArraySet;
21
22 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimExternalListener;
23 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimInternalListener;
24 import org.opendaylight.controller.protocol_plugin.openflow.IOFStatisticsListener;
25 import org.opendaylight.controller.protocol_plugin.openflow.core.IController;
26 import org.opendaylight.controller.protocol_plugin.openflow.core.IMessageListener;
27 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitch;
28 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitchStateListener;
29 import org.opendaylight.controller.sal.connection.IPluginOutConnectionService;
30 import org.opendaylight.controller.sal.core.Actions;
31 import org.opendaylight.controller.sal.core.Buffers;
32 import org.opendaylight.controller.sal.core.Capabilities;
33 import org.opendaylight.controller.sal.core.ContainerFlow;
34 import org.opendaylight.controller.sal.core.Description;
35 import org.opendaylight.controller.sal.core.IContainerAware;
36 import org.opendaylight.controller.sal.core.IContainerListener;
37 import org.opendaylight.controller.sal.core.MacAddress;
38 import org.opendaylight.controller.sal.core.Node;
39 import org.opendaylight.controller.sal.core.NodeConnector;
40 import org.opendaylight.controller.sal.core.Property;
41 import org.opendaylight.controller.sal.core.Tables;
42 import org.opendaylight.controller.sal.core.TimeStamp;
43 import org.opendaylight.controller.sal.core.UpdateType;
44 import org.opendaylight.controller.sal.utils.GlobalConstants;
45 import org.opendaylight.controller.sal.utils.NodeCreator;
46 import org.openflow.protocol.OFMessage;
47 import org.openflow.protocol.OFPortStatus;
48 import org.openflow.protocol.OFPortStatus.OFPortReason;
49 import org.openflow.protocol.OFType;
50 import org.openflow.protocol.statistics.OFDescriptionStatistics;
51 import org.openflow.protocol.statistics.OFStatistics;
52 import org.slf4j.Logger;
53 import org.slf4j.LoggerFactory;
54
55 /**
56  * The class describes a shim layer that bridges inventory events from Openflow
57  * core to various listeners. The notifications are filtered based on container
58  * configurations.
59  *
60  *
61  */
62 public class InventoryServiceShim implements IContainerListener,
63         IMessageListener, ISwitchStateListener, IOFStatisticsListener, IContainerAware {
64     protected static final Logger logger = LoggerFactory
65             .getLogger(InventoryServiceShim.class);
66     private IController controller = null;
67     private final ConcurrentMap<String, IInventoryShimInternalListener> inventoryShimInternalListeners = new ConcurrentHashMap<String, IInventoryShimInternalListener>();
68     private final Set<IInventoryShimInternalListener> globalInventoryShimInternalListeners = new HashSet<IInventoryShimInternalListener>();
69     private final List<IInventoryShimExternalListener> inventoryShimExternalListeners = new CopyOnWriteArrayList<IInventoryShimExternalListener>();
70     private final ConcurrentMap<NodeConnector, Set<String>> nodeConnectorContainerMap = new ConcurrentHashMap<NodeConnector, Set<String>>();
71     private final ConcurrentMap<Node, Set<String>> nodeContainerMap = new ConcurrentHashMap<Node, Set<String>>();
72     private final ConcurrentMap<NodeConnector, Set<Property>> nodeConnectorProps = new ConcurrentHashMap<NodeConnector, Set<Property>>();
73     private final ConcurrentMap<Node, Set<Property>> nodeProps = new ConcurrentHashMap<Node, Set<Property>>();
74     private IPluginOutConnectionService connectionOutService;
75
76     void setController(IController s) {
77         this.controller = s;
78     }
79
80     void unsetController(IController s) {
81         if (this.controller == s) {
82             this.controller = null;
83         }
84     }
85
86     void setInventoryShimGlobalInternalListener(Map<?, ?> props,
87             IInventoryShimInternalListener s) {
88         if ((this.globalInventoryShimInternalListeners != null)) {
89             this.globalInventoryShimInternalListeners.add(s);
90         }
91     }
92
93     void unsetInventoryShimGlobalInternalListener(Map<?, ?> props,
94             IInventoryShimInternalListener s) {
95         if ((this.globalInventoryShimInternalListeners != null)) {
96             this.globalInventoryShimInternalListeners.remove(s);
97         }
98     }
99
100     void setInventoryShimInternalListener(Map<?, ?> props,
101             IInventoryShimInternalListener s) {
102         if (props == null) {
103             logger.error("setInventoryShimInternalListener property is null");
104             return;
105         }
106         String containerName = (String) props.get("containerName");
107         if (containerName == null) {
108             logger.error("setInventoryShimInternalListener containerName not supplied");
109             return;
110         }
111         if ((this.inventoryShimInternalListeners != null)
112                 && !this.inventoryShimInternalListeners.containsValue(s)) {
113             this.inventoryShimInternalListeners.put(containerName, s);
114             logger.trace(
115                     "Added inventoryShimInternalListener for container {}",
116                     containerName);
117         }
118     }
119
120     void unsetInventoryShimInternalListener(Map<?, ?> props,
121             IInventoryShimInternalListener s) {
122         if (props == null) {
123             logger.error("unsetInventoryShimInternalListener property is null");
124             return;
125         }
126         String containerName = (String) props.get("containerName");
127         if (containerName == null) {
128             logger.error("setInventoryShimInternalListener containerName not supplied");
129             return;
130         }
131         if ((this.inventoryShimInternalListeners != null)
132                 && this.inventoryShimInternalListeners.get(containerName) != null
133                 && this.inventoryShimInternalListeners.get(containerName)
134                         .equals(s)) {
135             this.inventoryShimInternalListeners.remove(containerName);
136             logger.trace(
137                     "Removed inventoryShimInternalListener for container {}",
138                     containerName);
139         }
140     }
141
142     void setInventoryShimExternalListener(IInventoryShimExternalListener s) {
143         logger.trace("Set inventoryShimExternalListener {}", s);
144         if ((this.inventoryShimExternalListeners != null)
145                 && !this.inventoryShimExternalListeners.contains(s)) {
146             this.inventoryShimExternalListeners.add(s);
147         }
148     }
149
150     void unsetInventoryShimExternalListener(IInventoryShimExternalListener s) {
151         logger.trace("Unset inventoryShimExternalListener {}", s);
152         if ((this.inventoryShimExternalListeners != null)
153                 && this.inventoryShimExternalListeners.contains(s)) {
154             this.inventoryShimExternalListeners.remove(s);
155         }
156     }
157
158     void setIPluginOutConnectionService(IPluginOutConnectionService s) {
159         connectionOutService = s;
160     }
161
162     void unsetIPluginOutConnectionService(IPluginOutConnectionService s) {
163         if (connectionOutService == s) {
164             connectionOutService = null;
165         }
166     }
167
168     /**
169      * Function called by the dependency manager when all the required
170      * dependencies are satisfied
171      *
172      */
173     void init() {
174         this.controller.addMessageListener(OFType.PORT_STATUS, this);
175         this.controller.addSwitchStateListener(this);
176     }
177
178     /**
179      * Function called after registering the service in OSGi service registry.
180      */
181     void started() {
182         /* Start with existing switches */
183         startService();
184     }
185
186     /**
187      * Function called by the dependency manager when at least one dependency
188      * become unsatisfied or when the component is shutting down because for
189      * example bundle is being stopped.
190      *
191      */
192     void destroy() {
193         this.controller.removeMessageListener(OFType.PACKET_IN, this);
194         this.controller.removeSwitchStateListener(this);
195
196         this.inventoryShimInternalListeners.clear();
197         this.nodeConnectorContainerMap.clear();
198         this.nodeContainerMap.clear();
199         this.globalInventoryShimInternalListeners.clear();
200         this.controller = null;
201     }
202
203     @Override
204     public void receive(ISwitch sw, OFMessage msg) {
205         if (msg instanceof OFPortStatus) {
206             handlePortStatusMessage(sw, (OFPortStatus) msg);
207         }
208         return;
209     }
210
211     protected void handlePortStatusMessage(ISwitch sw, OFPortStatus m) {
212         Node node = NodeCreator.createOFNode(sw.getId());
213         NodeConnector nodeConnector = PortConverter.toNodeConnector(
214             m.getDesc().getPortNumber(), node);
215         // get node connector properties
216         Set<Property> props = InventoryServiceHelper.OFPortToProps(m.getDesc());
217
218         UpdateType type = null;
219         if (m.getReason() == (byte) OFPortReason.OFPPR_ADD.ordinal()) {
220             type = UpdateType.ADDED;
221             nodeConnectorProps.put(nodeConnector, props);
222         } else if (m.getReason() == (byte) OFPortReason.OFPPR_DELETE.ordinal()) {
223             type = UpdateType.REMOVED;
224             nodeConnectorProps.remove(nodeConnector);
225         } else if (m.getReason() == (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
226             type = UpdateType.CHANGED;
227             nodeConnectorProps.put(nodeConnector, props);
228         }
229
230         logger.trace("handlePortStatusMessage {} type {}", nodeConnector, type);
231
232         if (type != null) {
233             notifyInventoryShimListener(nodeConnector, type, props);
234         }
235     }
236
237     @Override
238     public void switchAdded(ISwitch sw) {
239         if (sw == null) {
240             return;
241         }
242         Node node = NodeCreator.createOFNode(sw.getId());
243         if ((nodeProps.get(node) != null)  && (connectionOutService.isLocal(node))) {
244             logger.debug("Ignore switchAdded {}", sw);
245             return;
246         }
247
248         // Add all the nodeConnectors of this switch
249         Map<NodeConnector, Set<Property>> ncProps = InventoryServiceHelper
250                 .OFSwitchToProps(sw);
251         for (Map.Entry<NodeConnector, Set<Property>> entry : ncProps.entrySet()) {
252             Set<Property> props = new HashSet<Property>();
253             Set<Property> prop = entry.getValue();
254             if (prop != null) {
255                 props.addAll(prop);
256             }
257             nodeConnectorProps.put(entry.getKey(), props);
258             notifyInventoryShimListener(entry.getKey(), UpdateType.ADDED, entry.getValue());
259         }
260
261         // Add this node
262         addNode(sw);
263     }
264
265     @Override
266     public void switchDeleted(ISwitch sw) {
267         if (sw == null) {
268             return;
269         }
270
271         removeNode(sw);
272     }
273
274     @Override
275     public void containerModeUpdated(UpdateType t) {
276         // do nothing
277     }
278
279     @Override
280     public void tagUpdated(String containerName, Node n, short oldTag,
281             short newTag, UpdateType t) {
282         logger.debug("tagUpdated: {} type {} for container {}", new Object[] {
283                 n, t, containerName });
284     }
285
286     @Override
287     public void containerFlowUpdated(String containerName,
288             ContainerFlow previousFlow, ContainerFlow currentFlow, UpdateType t) {
289     }
290
291     @Override
292     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
293         logger.debug("nodeConnectorUpdated: {} type {} for container {}", new Object[] { nc, t, containerName });
294         Node node = nc.getNode();
295         Set<String> ncContainers = this.nodeConnectorContainerMap.get(nc);
296         Set<String> nodeContainers = this.nodeContainerMap.get(node);
297         if (ncContainers == null) {
298             ncContainers = new CopyOnWriteArraySet<String>();
299         }
300         if (nodeContainers == null) {
301             nodeContainers = new CopyOnWriteArraySet<String>();
302         }
303         boolean notifyNodeUpdate = false;
304
305         switch (t) {
306         case ADDED:
307             if (ncContainers.add(containerName)) {
308                 this.nodeConnectorContainerMap.put(nc, ncContainers);
309             }
310             if (nodeContainers.add(containerName)) {
311                 this.nodeContainerMap.put(node, nodeContainers);
312                 notifyNodeUpdate = true;
313             }
314             break;
315         case REMOVED:
316             if (ncContainers.remove(containerName)) {
317                 if (ncContainers.isEmpty()) {
318                     // Do cleanup to reduce memory footprint if no
319                     // elements to be tracked
320                     this.nodeConnectorContainerMap.remove(nc);
321                 } else {
322                     this.nodeConnectorContainerMap.put(nc, ncContainers);
323                 }
324             }
325             boolean nodeContainerUpdate = true;
326             for (NodeConnector ncContainer : nodeConnectorContainerMap.keySet()) {
327                 if ((ncContainer.getNode().equals(node)) && (nodeConnectorContainerMap.get(ncContainer).contains(containerName))) {
328                     nodeContainerUpdate = false;
329                     break;
330                 }
331             }
332             if (nodeContainerUpdate) {
333                 nodeContainers.remove(containerName);
334                 notifyNodeUpdate = true;
335                 if (nodeContainers.isEmpty()) {
336                     this.nodeContainerMap.remove(node);
337                 } else {
338                     this.nodeContainerMap.put(node, nodeContainers);
339                 }
340             }
341             break;
342         case CHANGED:
343             break;
344         }
345
346         Set<Property> nodeProp = nodeProps.get(node);
347         if (nodeProp == null) {
348             return;
349         }
350         Set<Property> ncProp = nodeConnectorProps.get(nc);
351         // notify InventoryService
352         notifyInventoryShimInternalListener(containerName, nc, t, ncProp);
353
354         if (notifyNodeUpdate) {
355             notifyInventoryShimInternalListener(containerName, node, t, nodeProp);
356         }
357     }
358
359     private void notifyInventoryShimExternalListener(Node node, UpdateType type, Set<Property> props) {
360         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
361             s.updateNode(node, type, props);
362         }
363     }
364
365     private void notifyInventoryShimExternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
366         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
367             s.updateNodeConnector(nodeConnector, type, props);
368         }
369     }
370
371     private void notifyInventoryShimInternalListener(String container,
372             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
373         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners.get(container);
374         if (inventoryShimInternalListener != null) {
375             inventoryShimInternalListener.updateNodeConnector(nodeConnector, type, props);
376             logger.trace("notifyInventoryShimInternalListener {} type {} for container {}", new Object[] {
377                     nodeConnector, type, container });
378         }
379     }
380
381     /*
382      * Notify all internal and external listeners
383      */
384     private void notifyInventoryShimListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
385
386         //establish locality before notifying
387         boolean isNodeLocal;
388         if (type == UpdateType.REMOVED){
389             //if removing get the locality first
390             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
391             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
392         } else {
393             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
394             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
395         }
396
397         if (isNodeLocal) {
398             // notify other containers
399             Set<String> containers = (nodeConnectorContainerMap.get(nodeConnector) == null) ? new HashSet<String>()
400                     : new HashSet<String>(nodeConnectorContainerMap.get(nodeConnector));
401             containers.add(GlobalConstants.DEFAULT.toString());
402             for (String container : containers) {
403                 notifyInventoryShimInternalListener(container, nodeConnector, type, props);
404             }
405
406             // Notify plugin listeners (Discovery, DataPacket, OFstats etc.)
407             notifyInventoryShimExternalListener(nodeConnector, type, props);
408
409             logger.debug("Connection service accepted the inventory notification for {} {}", nodeConnector, type);
410         } else {
411             logger.debug("Connection service dropped the inventory notification for {} {}", nodeConnector, type);
412         }
413     }
414
415     /*
416      * Notify all internal and external listeners
417      */
418     private void notifyInventoryShimListener(Node node, UpdateType type, Set<Property> props) {
419
420         //establish locality before notifying
421         boolean isNodeLocal;
422         if (type == UpdateType.REMOVED){
423             //if removing get the locality first
424             isNodeLocal = connectionOutService.isLocal(node);
425             notifyGlobalInventoryShimInternalListener(node, type, props);
426         } else {
427             notifyGlobalInventoryShimInternalListener(node, type, props);
428             isNodeLocal = connectionOutService.isLocal(node);
429         }
430
431         if (isNodeLocal) {
432             // Now notify other containers
433             Set<String> containers = (nodeContainerMap.get(node) == null) ? new HashSet<String>()
434                     : new HashSet<String>(nodeContainerMap.get(node));
435             containers.add(GlobalConstants.DEFAULT.toString());
436             for (String container : containers) {
437                 notifyInventoryShimInternalListener(container, node, type, props);
438             }
439
440             // Notify plugin listeners (Discovery, DataPacket, OFstats etc.)
441             notifyInventoryShimExternalListener(node, type, props);
442
443             logger.debug("Connection service accepted the inventory notification for {} {}", node, type);
444         } else {
445             logger.debug("Connection service dropped the inventory notification for {} {}", node, type);
446         }
447     }
448
449     private void notifyGlobalInventoryShimInternalListener(Node node, UpdateType type, Set<Property> props) {
450         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
451             globalListener.updateNode(node, type, props);
452             logger.trace("notifyGlobalInventoryShimInternalListener {} type {}", new Object[] { node, type });
453         }
454     }
455
456     private void notifyGlobalInventoryShimInternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
457         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
458             globalListener.updateNodeConnector(nodeConnector, type, props);
459             logger.trace(
460                     "notifyGlobalInventoryShimInternalListener {} type {}",
461                     new Object[] { nodeConnector, type });
462         }
463     }
464
465     private void notifyInventoryShimInternalListener(String container,
466             Node node, UpdateType type, Set<Property> props) {
467         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
468                 .get(container);
469         if (inventoryShimInternalListener != null) {
470             inventoryShimInternalListener.updateNode(node, type, props);
471             logger.trace(
472                     "notifyInventoryShimInternalListener {} type {} for container {}",
473                     new Object[] { node, type, container });
474         }
475     }
476
477     private void addNode(ISwitch sw) {
478         Node node = NodeCreator.createOFNode(sw.getId());
479         UpdateType type = UpdateType.ADDED;
480
481         Set<Property> props = new HashSet<Property>();
482         Long sid = (Long) node.getID();
483
484         Date connectedSince = sw.getConnectedDate();
485         Long connectedSinceTime = (connectedSince == null) ? 0 : connectedSince
486                 .getTime();
487         props.add(new TimeStamp(connectedSinceTime, "connectedSince"));
488         props.add(new MacAddress(deriveMacAddress(sid)));
489
490         byte tables = sw.getTables();
491         Tables t = new Tables(tables);
492         if (t != null) {
493             props.add(t);
494         }
495         int cap = sw.getCapabilities();
496         Capabilities c = new Capabilities(cap);
497         if (c != null) {
498             props.add(c);
499         }
500         int act = sw.getActions();
501         Actions a = new Actions(act);
502         if (a != null) {
503             props.add(a);
504         }
505         int buffers = sw.getBuffers();
506         Buffers b = new Buffers(buffers);
507         if (b != null) {
508             props.add(b);
509         }
510
511         if ((nodeProps.get(node) == null) &&  (connectionOutService.isLocal(node)))  {
512             // The switch is connected for the first time, flush all flows
513             // that may exist on this switch
514             sw.deleteAllFlows();
515        }
516         nodeProps.put(node, props);
517         // Notify all internal and external listeners
518         notifyInventoryShimListener(node, type, props);
519     }
520
521     private void removeNode(ISwitch sw) {
522         Node node = NodeCreator.createOFNode(sw.getId());
523         if(node == null) {
524             return;
525         }
526         removeNodeConnectorProps(node);
527         nodeProps.remove(node);
528         UpdateType type = UpdateType.REMOVED;
529         // Notify all internal and external listeners
530         notifyInventoryShimListener(node, type, null);
531     }
532
533     private void startService() {
534         // Get a snapshot of all the existing switches
535         Map<Long, ISwitch> switches = this.controller.getSwitches();
536         for (ISwitch sw : switches.values()) {
537             switchAdded(sw);
538         }
539     }
540
541     private void removeNodeConnectorProps(Node node) {
542         List<NodeConnector> ncList = new ArrayList<NodeConnector>();
543         for (NodeConnector nc : nodeConnectorProps.keySet()) {
544             if (nc.getNode().equals(node)) {
545                 ncList.add(nc);
546             }
547         }
548         for (NodeConnector nc : ncList) {
549             nodeConnectorProps.remove(nc);
550         }
551     }
552
553     @Override
554     public void descriptionStatisticsRefreshed(Long switchId, List<OFStatistics> descriptionStats) {
555         Node node = NodeCreator.createOFNode(switchId);
556         Set<Property> properties = new HashSet<Property>(1);
557         OFDescriptionStatistics ofDesc = (OFDescriptionStatistics) descriptionStats.get(0);
558         Description desc = new Description(ofDesc.getDatapathDescription());
559         properties.add(desc);
560
561         // Notify all internal and external listeners
562         notifyInventoryShimListener(node, UpdateType.CHANGED, properties);
563     }
564
565     private byte[] deriveMacAddress(long dpid) {
566         byte[] mac = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
567
568         for (short i = 0; i < 6; i++) {
569             mac[5 - i] = (byte) dpid;
570             dpid >>= 8;
571         }
572
573         return mac;
574     }
575
576     @Override
577     public void flowStatisticsRefreshed(Long switchId, List<OFStatistics> flows) {
578         // Nothing to do
579     }
580
581     @Override
582     public void portStatisticsRefreshed(Long switchId, List<OFStatistics> ports) {
583         // Nothing to do
584     }
585
586     @Override
587     public void tableStatisticsRefreshed(Long switchId, List<OFStatistics> tables) {
588         // Nothing to do
589     }
590
591     @Override
592     public void containerCreate(String containerName) {
593         // Nothing to do
594     }
595
596     @Override
597     public void containerDestroy(String containerName) {
598         Set<NodeConnector> removeNodeConnectorSet = new HashSet<NodeConnector>();
599         Set<Node> removeNodeSet = new HashSet<Node>();
600         for (Map.Entry<NodeConnector, Set<String>> entry : nodeConnectorContainerMap.entrySet()) {
601             Set<String> ncContainers = entry.getValue();
602             if (ncContainers.contains(containerName)) {
603                 NodeConnector nodeConnector = entry.getKey();
604                 removeNodeConnectorSet.add(nodeConnector);
605             }
606         }
607         for (Map.Entry<Node, Set<String>> entry : nodeContainerMap.entrySet()) {
608             Set<String> nodeContainers = entry.getValue();
609             if (nodeContainers.contains(containerName)) {
610                 Node node = entry.getKey();
611                 removeNodeSet.add(node);
612             }
613         }
614         for (NodeConnector nodeConnector : removeNodeConnectorSet) {
615             Set<String> ncContainers = nodeConnectorContainerMap.get(nodeConnector);
616             ncContainers.remove(containerName);
617             if (ncContainers.isEmpty()) {
618                 nodeConnectorContainerMap.remove(nodeConnector);
619             }
620         }
621         for (Node node : removeNodeSet) {
622             Set<String> nodeContainers = nodeContainerMap.get(node);
623             nodeContainers.remove(containerName);
624             if (nodeContainers.isEmpty()) {
625                 nodeContainerMap.remove(node);
626             }
627         }
628     }
629 }