Merge "In o.o.c.switchmanager.SpanConfig, handle node connector string format correctly."
[controller.git] / opendaylight / protocol_plugins / openflow / src / main / java / org / opendaylight / controller / protocol_plugin / openflow / internal / InventoryServiceShim.java
1 /*
2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.protocol_plugin.openflow.internal;
10
11 import java.util.ArrayList;
12 import java.util.Date;
13 import java.util.HashSet;
14 import java.util.List;
15 import java.util.Map;
16 import java.util.Set;
17 import java.util.concurrent.ConcurrentHashMap;
18 import java.util.concurrent.ConcurrentMap;
19 import java.util.concurrent.CopyOnWriteArrayList;
20 import java.util.concurrent.CopyOnWriteArraySet;
21
22 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimExternalListener;
23 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimInternalListener;
24 import org.opendaylight.controller.protocol_plugin.openflow.IOFStatisticsListener;
25 import org.opendaylight.controller.protocol_plugin.openflow.core.IController;
26 import org.opendaylight.controller.protocol_plugin.openflow.core.IMessageListener;
27 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitch;
28 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitchStateListener;
29 import org.opendaylight.controller.sal.connection.IPluginOutConnectionService;
30 import org.opendaylight.controller.sal.core.Actions;
31 import org.opendaylight.controller.sal.core.Buffers;
32 import org.opendaylight.controller.sal.core.Capabilities;
33 import org.opendaylight.controller.sal.core.ContainerFlow;
34 import org.opendaylight.controller.sal.core.Description;
35 import org.opendaylight.controller.sal.core.IContainerAware;
36 import org.opendaylight.controller.sal.core.IContainerListener;
37 import org.opendaylight.controller.sal.core.MacAddress;
38 import org.opendaylight.controller.sal.core.Node;
39 import org.opendaylight.controller.sal.core.NodeConnector;
40 import org.opendaylight.controller.sal.core.Property;
41 import org.opendaylight.controller.sal.core.Tables;
42 import org.opendaylight.controller.sal.core.TimeStamp;
43 import org.opendaylight.controller.sal.core.UpdateType;
44 import org.opendaylight.controller.sal.utils.GlobalConstants;
45 import org.opendaylight.controller.sal.utils.NodeCreator;
46 import org.openflow.protocol.OFMessage;
47 import org.openflow.protocol.OFPortStatus;
48 import org.openflow.protocol.OFPortStatus.OFPortReason;
49 import org.openflow.protocol.OFType;
50 import org.openflow.protocol.statistics.OFDescriptionStatistics;
51 import org.openflow.protocol.statistics.OFStatistics;
52 import org.slf4j.Logger;
53 import org.slf4j.LoggerFactory;
54
55 /**
56  * The class describes a shim layer that bridges inventory events from Openflow
57  * core to various listeners. The notifications are filtered based on container
58  * configurations.
59  *
60  *
61  */
62 public class InventoryServiceShim implements IContainerListener,
63         IMessageListener, ISwitchStateListener, IOFStatisticsListener, IContainerAware {
64     protected static final Logger logger = LoggerFactory
65             .getLogger(InventoryServiceShim.class);
66     private IController controller = null;
67     private final ConcurrentMap<String, IInventoryShimInternalListener> inventoryShimInternalListeners = new ConcurrentHashMap<String, IInventoryShimInternalListener>();
68     private final Set<IInventoryShimInternalListener> globalInventoryShimInternalListeners = new HashSet<IInventoryShimInternalListener>();
69     private final List<IInventoryShimExternalListener> inventoryShimExternalListeners = new CopyOnWriteArrayList<IInventoryShimExternalListener>();
70     private final ConcurrentMap<NodeConnector, Set<String>> nodeConnectorContainerMap = new ConcurrentHashMap<NodeConnector, Set<String>>();
71     private final ConcurrentMap<Node, Set<String>> nodeContainerMap = new ConcurrentHashMap<Node, Set<String>>();
72     private final ConcurrentMap<NodeConnector, Set<Property>> nodeConnectorProps = new ConcurrentHashMap<NodeConnector, Set<Property>>();
73     private final ConcurrentMap<Node, Set<Property>> nodeProps = new ConcurrentHashMap<Node, Set<Property>>();
74     private IPluginOutConnectionService connectionOutService;
75
76     void setController(IController s) {
77         this.controller = s;
78     }
79
80     void unsetController(IController s) {
81         if (this.controller == s) {
82             this.controller = null;
83         }
84     }
85
86     void setInventoryShimGlobalInternalListener(Map<?, ?> props,
87             IInventoryShimInternalListener s) {
88         if ((this.globalInventoryShimInternalListeners != null)) {
89             this.globalInventoryShimInternalListeners.add(s);
90         }
91     }
92
93     void unsetInventoryShimGlobalInternalListener(Map<?, ?> props,
94             IInventoryShimInternalListener s) {
95         if ((this.globalInventoryShimInternalListeners != null)) {
96             this.globalInventoryShimInternalListeners.remove(s);
97         }
98     }
99
100     void setInventoryShimInternalListener(Map<?, ?> props,
101             IInventoryShimInternalListener s) {
102         if (props == null) {
103             logger.error("setInventoryShimInternalListener property is null");
104             return;
105         }
106         String containerName = (String) props.get("containerName");
107         if (containerName == null) {
108             logger.error("setInventoryShimInternalListener containerName not supplied");
109             return;
110         }
111         if ((this.inventoryShimInternalListeners != null)
112                 && !this.inventoryShimInternalListeners.containsValue(s)) {
113             this.inventoryShimInternalListeners.put(containerName, s);
114             logger.trace(
115                     "Added inventoryShimInternalListener for container {}",
116                     containerName);
117         }
118     }
119
120     void unsetInventoryShimInternalListener(Map<?, ?> props,
121             IInventoryShimInternalListener s) {
122         if (props == null) {
123             logger.error("unsetInventoryShimInternalListener property is null");
124             return;
125         }
126         String containerName = (String) props.get("containerName");
127         if (containerName == null) {
128             logger.error("setInventoryShimInternalListener containerName not supplied");
129             return;
130         }
131         if ((this.inventoryShimInternalListeners != null)
132                 && this.inventoryShimInternalListeners.get(containerName) != null
133                 && this.inventoryShimInternalListeners.get(containerName)
134                         .equals(s)) {
135             this.inventoryShimInternalListeners.remove(containerName);
136             logger.trace(
137                     "Removed inventoryShimInternalListener for container {}",
138                     containerName);
139         }
140     }
141
142     void setInventoryShimExternalListener(IInventoryShimExternalListener s) {
143         logger.trace("Set inventoryShimExternalListener {}", s);
144         if ((this.inventoryShimExternalListeners != null)
145                 && !this.inventoryShimExternalListeners.contains(s)) {
146             this.inventoryShimExternalListeners.add(s);
147         }
148     }
149
150     void unsetInventoryShimExternalListener(IInventoryShimExternalListener s) {
151         logger.trace("Unset inventoryShimExternalListener {}", s);
152         if ((this.inventoryShimExternalListeners != null)
153                 && this.inventoryShimExternalListeners.contains(s)) {
154             this.inventoryShimExternalListeners.remove(s);
155         }
156     }
157
158     void setIPluginOutConnectionService(IPluginOutConnectionService s) {
159         connectionOutService = s;
160     }
161
162     void unsetIPluginOutConnectionService(IPluginOutConnectionService s) {
163         if (connectionOutService == s) {
164             connectionOutService = null;
165         }
166     }
167
168     /**
169      * Function called by the dependency manager when all the required
170      * dependencies are satisfied
171      *
172      */
173     void init() {
174         this.controller.addMessageListener(OFType.PORT_STATUS, this);
175         this.controller.addSwitchStateListener(this);
176     }
177
178     /**
179      * Function called after registering the service in OSGi service registry.
180      */
181     void started() {
182         /* Start with existing switches */
183         startService();
184     }
185
186     /**
187      * Function called by the dependency manager when at least one dependency
188      * become unsatisfied or when the component is shutting down because for
189      * example bundle is being stopped.
190      *
191      */
192     void destroy() {
193         this.controller.removeMessageListener(OFType.PACKET_IN, this);
194         this.controller.removeSwitchStateListener(this);
195
196         this.inventoryShimInternalListeners.clear();
197         this.nodeConnectorContainerMap.clear();
198         this.nodeContainerMap.clear();
199         this.globalInventoryShimInternalListeners.clear();
200         this.controller = null;
201     }
202
203     @Override
204     public void receive(ISwitch sw, OFMessage msg) {
205         if (msg instanceof OFPortStatus) {
206             handlePortStatusMessage(sw, (OFPortStatus) msg);
207         }
208         return;
209     }
210
211     protected void handlePortStatusMessage(ISwitch sw, OFPortStatus m) {
212         Node node = NodeCreator.createOFNode(sw.getId());
213         NodeConnector nodeConnector = PortConverter.toNodeConnector(
214             m.getDesc().getPortNumber(), node);
215         // get node connector properties
216         Set<Property> props = InventoryServiceHelper.OFPortToProps(m.getDesc());
217
218         UpdateType type = null;
219         if (m.getReason() == (byte) OFPortReason.OFPPR_ADD.ordinal()) {
220             type = UpdateType.ADDED;
221             nodeConnectorProps.put(nodeConnector, props);
222         } else if (m.getReason() == (byte) OFPortReason.OFPPR_DELETE.ordinal()) {
223             type = UpdateType.REMOVED;
224             nodeConnectorProps.remove(nodeConnector);
225         } else if (m.getReason() == (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
226             type = UpdateType.CHANGED;
227             nodeConnectorProps.put(nodeConnector, props);
228         }
229
230         logger.trace("handlePortStatusMessage {} type {}", nodeConnector, type);
231
232         if (type != null) {
233             notifyInventoryShimListener(nodeConnector, type, props);
234         }
235     }
236
237     @Override
238     public void switchAdded(ISwitch sw) {
239         if (sw == null) {
240             return;
241         }
242         Node node = NodeCreator.createOFNode(sw.getId());
243         if ((nodeProps.get(node) != null)  && (connectionOutService.isLocal(node))) {
244             logger.debug("Ignore switchAdded {}", sw);
245             return;
246         }
247
248         // Add all the nodeConnectors of this switch
249         Map<NodeConnector, Set<Property>> ncProps = InventoryServiceHelper
250                 .OFSwitchToProps(sw);
251         for (Map.Entry<NodeConnector, Set<Property>> entry : ncProps.entrySet()) {
252             Set<Property> props = new HashSet<Property>();
253             Set<Property> prop = entry.getValue();
254             if (prop != null) {
255                 props.addAll(prop);
256             }
257             nodeConnectorProps.put(entry.getKey(), props);
258             notifyInventoryShimListener(entry.getKey(), UpdateType.ADDED,
259                     entry.getValue());
260         }
261
262         // Add this node
263         addNode(sw);
264     }
265
266     @Override
267     public void switchDeleted(ISwitch sw) {
268         if (sw == null) {
269             return;
270         }
271
272         removeNode(sw);
273     }
274
275     @Override
276     public void containerModeUpdated(UpdateType t) {
277         // do nothing
278     }
279
280     @Override
281     public void tagUpdated(String containerName, Node n, short oldTag,
282             short newTag, UpdateType t) {
283         logger.debug("tagUpdated: {} type {} for container {}", new Object[] {
284                 n, t, containerName });
285     }
286
287     @Override
288     public void containerFlowUpdated(String containerName,
289             ContainerFlow previousFlow, ContainerFlow currentFlow, UpdateType t) {
290     }
291
292     @Override
293     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
294         logger.debug("nodeConnectorUpdated: {} type {} for container {}", new Object[] { nc, t, containerName });
295         Node node = nc.getNode();
296         Set<String> ncContainers = this.nodeConnectorContainerMap.get(nc);
297         Set<String> nodeContainers = this.nodeContainerMap.get(node);
298         if (ncContainers == null) {
299             ncContainers = new CopyOnWriteArraySet<String>();
300         }
301         if (nodeContainers == null) {
302             nodeContainers = new CopyOnWriteArraySet<String>();
303         }
304         boolean notifyNodeUpdate = false;
305
306         switch (t) {
307         case ADDED:
308             if (ncContainers.add(containerName)) {
309                 this.nodeConnectorContainerMap.put(nc, ncContainers);
310             }
311             if (nodeContainers.add(containerName)) {
312                 this.nodeContainerMap.put(node, nodeContainers);
313                 notifyNodeUpdate = true;
314             }
315             break;
316         case REMOVED:
317             if (ncContainers.remove(containerName)) {
318                 if (ncContainers.isEmpty()) {
319                     // Do cleanup to reduce memory footprint if no
320                     // elements to be tracked
321                     this.nodeConnectorContainerMap.remove(nc);
322                 } else {
323                     this.nodeConnectorContainerMap.put(nc, ncContainers);
324                 }
325             }
326             boolean nodeContainerUpdate = true;
327             for (NodeConnector ncContainer : nodeConnectorContainerMap.keySet()) {
328                 if ((ncContainer.getNode().equals(node)) && (nodeConnectorContainerMap.get(ncContainer).contains(containerName))) {
329                     nodeContainerUpdate = false;
330                     break;
331                 }
332             }
333             if (nodeContainerUpdate) {
334                 nodeContainers.remove(containerName);
335                 notifyNodeUpdate = true;
336                 if (nodeContainers.isEmpty()) {
337                     this.nodeContainerMap.remove(node);
338                 } else {
339                     this.nodeContainerMap.put(node, nodeContainers);
340                 }
341             }
342             break;
343         case CHANGED:
344             break;
345         }
346
347         Set<Property> nodeProp = nodeProps.get(node);
348         if (nodeProp == null) {
349             return;
350         }
351         Set<Property> ncProp = nodeConnectorProps.get(nc);
352         // notify InventoryService
353         notifyInventoryShimInternalListener(containerName, nc, t, ncProp);
354
355         if (notifyNodeUpdate) {
356             notifyInventoryShimInternalListener(containerName, node, t, nodeProp);
357         }
358     }
359
360     private void notifyInventoryShimExternalListener(Node node,
361             UpdateType type, Set<Property> props) {
362         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
363             s.updateNode(node, type, props);
364         }
365     }
366
367     private void notifyInventoryShimExternalListener(
368             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
369         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
370             s.updateNodeConnector(nodeConnector, type, props);
371         }
372     }
373
374     private void notifyInventoryShimInternalListener(String container,
375             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
376         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
377                 .get(container);
378         if (inventoryShimInternalListener != null) {
379             inventoryShimInternalListener.updateNodeConnector(nodeConnector,
380                     type, props);
381             logger.trace(
382                     "notifyInventoryShimInternalListener {} type {} for container {}",
383                     new Object[] { nodeConnector, type, container });
384         }
385     }
386
387     /*
388      * Notify all internal and external listeners
389      */
390     private void notifyInventoryShimListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
391
392         //establish locality before notifying
393         boolean isNodeLocal;
394         if (type == UpdateType.REMOVED){
395             //if removing get the locality first
396             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
397             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
398         } else {
399             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
400             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
401         }
402
403         if (isNodeLocal) {
404             // notify other containers
405             Set<String> containers = (nodeConnectorContainerMap.get(nodeConnector) == null) ? new HashSet<String>()
406                     : new HashSet<String>(nodeConnectorContainerMap.get(nodeConnector));
407             containers.add(GlobalConstants.DEFAULT.toString());
408             for (String container : containers) {
409                 notifyInventoryShimInternalListener(container, nodeConnector, type, props);
410             }
411
412             // Notify DiscoveryService
413             notifyInventoryShimExternalListener(nodeConnector, type, props);
414
415             logger.debug("Connection service accepted the inventory notification for {} {}", nodeConnector, type);
416         } else {
417             logger.debug("Connection service dropped the inventory notification for {} {}", nodeConnector, type);
418         }
419     }
420
421     /*
422      * Notify all internal and external listeners
423      */
424     private void notifyInventoryShimListener(Node node, UpdateType type, Set<Property> props) {
425
426         //establish locality before notifying
427         boolean isNodeLocal;
428         if (type == UpdateType.REMOVED){
429             //if removing get the locality first
430             isNodeLocal = connectionOutService.isLocal(node);
431             notifyGlobalInventoryShimInternalListener(node, type, props);
432         } else {
433             notifyGlobalInventoryShimInternalListener(node, type, props);
434             isNodeLocal = connectionOutService.isLocal(node);
435         }
436
437         if (isNodeLocal) {
438             // Now notify other containers
439             Set<String> containers = (nodeContainerMap.get(node) == null) ? new HashSet<String>() : new HashSet<String>(
440                     nodeContainerMap.get(node));
441             containers.add(GlobalConstants.DEFAULT.toString());
442             for (String container : containers) {
443                 notifyInventoryShimInternalListener(container, node, type, props);
444             }
445             // Notify external listener
446             notifyInventoryShimExternalListener(node, type, props);
447
448             logger.debug("Connection service accepted the inventory notification for {} {}", node, type);
449         } else {
450             logger.debug("Connection service dropped the inventory notification for {} {}", node, type);
451         }
452     }
453
454     private void notifyGlobalInventoryShimInternalListener(Node node, UpdateType type, Set<Property> props) {
455         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
456             globalListener.updateNode(node, type, props);
457             logger.trace(
458                     "notifyGlobalInventoryShimInternalListener {} type {}",
459                     new Object[] { node, type });
460         }
461     }
462
463     private void notifyGlobalInventoryShimInternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
464         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
465             globalListener.updateNodeConnector(nodeConnector, type, props);
466             logger.trace(
467                     "notifyGlobalInventoryShimInternalListener {} type {}",
468                     new Object[] { nodeConnector, type });
469         }
470     }
471
472     private void notifyInventoryShimInternalListener(String container,
473             Node node, UpdateType type, Set<Property> props) {
474         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
475                 .get(container);
476         if (inventoryShimInternalListener != null) {
477             inventoryShimInternalListener.updateNode(node, type, props);
478             logger.trace(
479                     "notifyInventoryShimInternalListener {} type {} for container {}",
480                     new Object[] { node, type, container });
481         }
482     }
483
484     private void addNode(ISwitch sw) {
485         Node node = NodeCreator.createOFNode(sw.getId());
486         UpdateType type = UpdateType.ADDED;
487
488         Set<Property> props = new HashSet<Property>();
489         Long sid = (Long) node.getID();
490
491         Date connectedSince = sw.getConnectedDate();
492         Long connectedSinceTime = (connectedSince == null) ? 0 : connectedSince
493                 .getTime();
494         props.add(new TimeStamp(connectedSinceTime, "connectedSince"));
495         props.add(new MacAddress(deriveMacAddress(sid)));
496
497         byte tables = sw.getTables();
498         Tables t = new Tables(tables);
499         if (t != null) {
500             props.add(t);
501         }
502         int cap = sw.getCapabilities();
503         Capabilities c = new Capabilities(cap);
504         if (c != null) {
505             props.add(c);
506         }
507         int act = sw.getActions();
508         Actions a = new Actions(act);
509         if (a != null) {
510             props.add(a);
511         }
512         int buffers = sw.getBuffers();
513         Buffers b = new Buffers(buffers);
514         if (b != null) {
515             props.add(b);
516         }
517
518         if ((nodeProps.get(node) == null) &&  (connectionOutService.isLocal(node)))  {
519             // The switch is connected for the first time, flush all flows
520             // that may exist on this switch
521             sw.deleteAllFlows();
522        }
523         nodeProps.put(node, props);
524         // Notify all internal and external listeners
525         notifyInventoryShimListener(node, type, props);
526     }
527
528     private void removeNode(ISwitch sw) {
529         Node node = NodeCreator.createOFNode(sw.getId());
530         if(node == null) {
531             return;
532         }
533         removeNodeConnectorProps(node);
534         nodeProps.remove(node);
535         UpdateType type = UpdateType.REMOVED;
536         // Notify all internal and external listeners
537         notifyInventoryShimListener(node, type, null);
538     }
539
540     private void startService() {
541         // Get a snapshot of all the existing switches
542         Map<Long, ISwitch> switches = this.controller.getSwitches();
543         for (ISwitch sw : switches.values()) {
544             switchAdded(sw);
545         }
546     }
547
548     private void removeNodeConnectorProps(Node node) {
549         List<NodeConnector> ncList = new ArrayList<NodeConnector>();
550         for (NodeConnector nc : nodeConnectorProps.keySet()) {
551             if (nc.getNode().equals(node)) {
552                 ncList.add(nc);
553             }
554         }
555         for (NodeConnector nc : ncList) {
556             nodeConnectorProps.remove(nc);
557         }
558     }
559
560     @Override
561     public void descriptionStatisticsRefreshed(Long switchId, List<OFStatistics> descriptionStats) {
562         Node node = NodeCreator.createOFNode(switchId);
563         Set<Property> properties = new HashSet<Property>(1);
564         OFDescriptionStatistics ofDesc = (OFDescriptionStatistics) descriptionStats.get(0);
565         Description desc = new Description(ofDesc.getDatapathDescription());
566         properties.add(desc);
567
568         // Notify all internal and external listeners
569         notifyInventoryShimListener(node, UpdateType.CHANGED, properties);
570     }
571
572     private byte[] deriveMacAddress(long dpid) {
573         byte[] mac = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
574
575         for (short i = 0; i < 6; i++) {
576             mac[5 - i] = (byte) dpid;
577             dpid >>= 8;
578         }
579
580         return mac;
581     }
582
583     @Override
584     public void flowStatisticsRefreshed(Long switchId, List<OFStatistics> flows) {
585         // Nothing to do
586     }
587
588     @Override
589     public void portStatisticsRefreshed(Long switchId, List<OFStatistics> ports) {
590         // Nothing to do
591     }
592
593     @Override
594     public void tableStatisticsRefreshed(Long switchId, List<OFStatistics> tables) {
595         // Nothing to do
596     }
597
598     @Override
599     public void containerCreate(String containerName) {
600         // Nothing to do
601     }
602
603     @Override
604     public void containerDestroy(String containerName) {
605         Set<NodeConnector> removeNodeConnectorSet = new HashSet<NodeConnector>();
606         Set<Node> removeNodeSet = new HashSet<Node>();
607         for (Map.Entry<NodeConnector, Set<String>> entry : nodeConnectorContainerMap.entrySet()) {
608             Set<String> ncContainers = entry.getValue();
609             if (ncContainers.contains(containerName)) {
610                 NodeConnector nodeConnector = entry.getKey();
611                 removeNodeConnectorSet.add(nodeConnector);
612             }
613         }
614         for (Map.Entry<Node, Set<String>> entry : nodeContainerMap.entrySet()) {
615             Set<String> nodeContainers = entry.getValue();
616             if (nodeContainers.contains(containerName)) {
617                 Node node = entry.getKey();
618                 removeNodeSet.add(node);
619             }
620         }
621         for (NodeConnector nodeConnector : removeNodeConnectorSet) {
622             Set<String> ncContainers = nodeConnectorContainerMap.get(nodeConnector);
623             ncContainers.remove(containerName);
624             if (ncContainers.isEmpty()) {
625                 nodeConnectorContainerMap.remove(nodeConnector);
626             }
627         }
628         for (Node node : removeNodeSet) {
629             Set<String> nodeContainers = nodeContainerMap.get(node);
630             nodeContainers.remove(containerName);
631             if (nodeContainers.isEmpty()) {
632                 nodeContainerMap.remove(node);
633             }
634         }
635     }
636 }