opendaylight/protocol_plugins/openflow/src/main/java/org/opendaylight/controller/protocol_plugin/openflow/internal/InventoryServiceShim.java

   1 /*
   2  * Copyright (c) 2013 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.protocol_plugin.openflow.internal;
  10
  11 import java.util.ArrayList;
  12 import java.util.Date;
  13 import java.util.HashSet;
  14 import java.util.List;
  15 import java.util.Map;
  16 import java.util.Set;
  17 import java.util.concurrent.ConcurrentHashMap;
  18 import java.util.concurrent.ConcurrentMap;
  19 import java.util.concurrent.CopyOnWriteArrayList;
  20 import java.util.concurrent.CopyOnWriteArraySet;
  21
  22 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimExternalListener;
  23 import org.opendaylight.controller.protocol_plugin.openflow.IInventoryShimInternalListener;
  24 import org.opendaylight.controller.protocol_plugin.openflow.IOFStatisticsListener;
  25 import org.opendaylight.controller.protocol_plugin.openflow.core.IController;
  26 import org.opendaylight.controller.protocol_plugin.openflow.core.IMessageListener;
  27 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitch;
  28 import org.opendaylight.controller.protocol_plugin.openflow.core.ISwitchStateListener;
  29 import org.opendaylight.controller.sal.connection.IPluginOutConnectionService;
  30 import org.opendaylight.controller.sal.core.Actions;
  31 import org.opendaylight.controller.sal.core.Buffers;
  32 import org.opendaylight.controller.sal.core.Capabilities;
  33 import org.opendaylight.controller.sal.core.ContainerFlow;
  34 import org.opendaylight.controller.sal.core.Description;
  35 import org.opendaylight.controller.sal.core.IContainerAware;
  36 import org.opendaylight.controller.sal.core.IContainerListener;
  37 import org.opendaylight.controller.sal.core.MacAddress;
  38 import org.opendaylight.controller.sal.core.Node;
  39 import org.opendaylight.controller.sal.core.NodeConnector;
  40 import org.opendaylight.controller.sal.core.Property;
  41 import org.opendaylight.controller.sal.core.Tables;
  42 import org.opendaylight.controller.sal.core.TimeStamp;
  43 import org.opendaylight.controller.sal.core.UpdateType;
  44 import org.opendaylight.controller.sal.utils.GlobalConstants;
  45 import org.opendaylight.controller.sal.utils.NodeCreator;
  46 import org.openflow.protocol.OFMessage;
  47 import org.openflow.protocol.OFPortStatus;
  48 import org.openflow.protocol.OFPortStatus.OFPortReason;
  49 import org.openflow.protocol.OFType;
  50 import org.openflow.protocol.statistics.OFDescriptionStatistics;
  51 import org.openflow.protocol.statistics.OFStatistics;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54
  55 /**
  56  * The class describes a shim layer that bridges inventory events from Openflow
  57  * core to various listeners. The notifications are filtered based on container
  58  * configurations.
  59  *
  60  *
  61  */
  62 public class InventoryServiceShim implements IContainerListener,
  63         IMessageListener, ISwitchStateListener, IOFStatisticsListener, IContainerAware {
  64     protected static final Logger logger = LoggerFactory
  65             .getLogger(InventoryServiceShim.class);
  66     private IController controller = null;
  67     private final ConcurrentMap<String, IInventoryShimInternalListener> inventoryShimInternalListeners = new ConcurrentHashMap<String, IInventoryShimInternalListener>();
  68     private final Set<IInventoryShimInternalListener> globalInventoryShimInternalListeners = new HashSet<IInventoryShimInternalListener>();
  69     private final List<IInventoryShimExternalListener> inventoryShimExternalListeners = new CopyOnWriteArrayList<IInventoryShimExternalListener>();
  70     private final ConcurrentMap<NodeConnector, Set<String>> nodeConnectorContainerMap = new ConcurrentHashMap<NodeConnector, Set<String>>();
  71     private final ConcurrentMap<Node, Set<String>> nodeContainerMap = new ConcurrentHashMap<Node, Set<String>>();
  72     private final ConcurrentMap<NodeConnector, Set<Property>> nodeConnectorProps = new ConcurrentHashMap<NodeConnector, Set<Property>>();
  73     private final ConcurrentMap<Node, Set<Property>> nodeProps = new ConcurrentHashMap<Node, Set<Property>>();
  74     private IPluginOutConnectionService connectionOutService;
  75
  76     void setController(IController s) {
  77         this.controller = s;
  78     }
  79
  80     void unsetController(IController s) {
  81         if (this.controller == s) {
  82             this.controller = null;
  83         }
  84     }
  85
  86     void setInventoryShimGlobalInternalListener(Map<?, ?> props,
  87             IInventoryShimInternalListener s) {
  88         if ((this.globalInventoryShimInternalListeners != null)) {
  89             this.globalInventoryShimInternalListeners.add(s);
  90         }
  91     }
  92
  93     void unsetInventoryShimGlobalInternalListener(Map<?, ?> props,
  94             IInventoryShimInternalListener s) {
  95         if ((this.globalInventoryShimInternalListeners != null)) {
  96             this.globalInventoryShimInternalListeners.remove(s);
  97         }
  98     }
  99
 100     void setInventoryShimInternalListener(Map<?, ?> props,
 101             IInventoryShimInternalListener s) {
 102         if (props == null) {
 103             logger.error("setInventoryShimInternalListener property is null");
 104             return;
 105         }
 106         String containerName = (String) props.get("containerName");
 107         if (containerName == null) {
 108             logger.error("setInventoryShimInternalListener containerName not supplied");
 109             return;
 110         }
 111         if ((this.inventoryShimInternalListeners != null)
 112                 && !this.inventoryShimInternalListeners.containsValue(s)) {
 113             this.inventoryShimInternalListeners.put(containerName, s);
 114             logger.trace(
 115                     "Added inventoryShimInternalListener for container {}",
 116                     containerName);
 117         }
 118     }
 119
 120     void unsetInventoryShimInternalListener(Map<?, ?> props,
 121             IInventoryShimInternalListener s) {
 122         if (props == null) {
 123             logger.error("unsetInventoryShimInternalListener property is null");
 124             return;
 125         }
 126         String containerName = (String) props.get("containerName");
 127         if (containerName == null) {
 128             logger.error("setInventoryShimInternalListener containerName not supplied");
 129             return;
 130         }
 131         if ((this.inventoryShimInternalListeners != null)
 132                 && this.inventoryShimInternalListeners.get(containerName) != null
 133                 && this.inventoryShimInternalListeners.get(containerName)
 134                         .equals(s)) {
 135             this.inventoryShimInternalListeners.remove(containerName);
 136             logger.trace(
 137                     "Removed inventoryShimInternalListener for container {}",
 138                     containerName);
 139         }
 140     }
 141
 142     void setInventoryShimExternalListener(IInventoryShimExternalListener s) {
 143         logger.trace("Set inventoryShimExternalListener {}", s);
 144         if ((this.inventoryShimExternalListeners != null)
 145                 && !this.inventoryShimExternalListeners.contains(s)) {
 146             this.inventoryShimExternalListeners.add(s);
 147         }
 148     }
 149
 150     void unsetInventoryShimExternalListener(IInventoryShimExternalListener s) {
 151         logger.trace("Unset inventoryShimExternalListener {}", s);
 152         if ((this.inventoryShimExternalListeners != null)
 153                 && this.inventoryShimExternalListeners.contains(s)) {
 154             this.inventoryShimExternalListeners.remove(s);
 155         }
 156     }
 157
 158     void setIPluginOutConnectionService(IPluginOutConnectionService s) {
 159         connectionOutService = s;
 160     }
 161
 162     void unsetIPluginOutConnectionService(IPluginOutConnectionService s) {
 163         if (connectionOutService == s) {
 164             connectionOutService = null;
 165         }
 166     }
 167
 168     /**
 169      * Function called by the dependency manager when all the required
 170      * dependencies are satisfied
 171      *
 172      */
 173     void init() {
 174         this.controller.addMessageListener(OFType.PORT_STATUS, this);
 175         this.controller.addSwitchStateListener(this);
 176     }
 177
 178     /**
 179      * Function called after registering the service in OSGi service registry.
 180      */
 181     void started() {
 182         /* Start with existing switches */
 183         startService();
 184     }
 185
 186     /**
 187      * Function called by the dependency manager when at least one dependency
 188      * become unsatisfied or when the component is shutting down because for
 189      * example bundle is being stopped.
 190      *
 191      */
 192     void destroy() {
 193         this.controller.removeMessageListener(OFType.PACKET_IN, this);
 194         this.controller.removeSwitchStateListener(this);
 195
 196         this.inventoryShimInternalListeners.clear();
 197         this.nodeConnectorContainerMap.clear();
 198         this.nodeContainerMap.clear();
 199         this.globalInventoryShimInternalListeners.clear();
 200         this.controller = null;
 201     }
 202
 203     @Override
 204     public void receive(ISwitch sw, OFMessage msg) {
 205         if (msg instanceof OFPortStatus) {
 206             handlePortStatusMessage(sw, (OFPortStatus) msg);
 207         }
 208         return;
 209     }
 210
 211     protected void handlePortStatusMessage(ISwitch sw, OFPortStatus m) {
 212         Node node = NodeCreator.createOFNode(sw.getId());
 213         NodeConnector nodeConnector = PortConverter.toNodeConnector(
 214             m.getDesc().getPortNumber(), node);
 215         // get node connector properties
 216         Set<Property> props = InventoryServiceHelper.OFPortToProps(m.getDesc());
 217
 218         UpdateType type = null;
 219         if (m.getReason() == (byte) OFPortReason.OFPPR_ADD.ordinal()) {
 220             type = UpdateType.ADDED;
 221             nodeConnectorProps.put(nodeConnector, props);
 222         } else if (m.getReason() == (byte) OFPortReason.OFPPR_DELETE.ordinal()) {
 223             type = UpdateType.REMOVED;
 224             nodeConnectorProps.remove(nodeConnector);
 225         } else if (m.getReason() == (byte) OFPortReason.OFPPR_MODIFY.ordinal()) {
 226             type = UpdateType.CHANGED;
 227             nodeConnectorProps.put(nodeConnector, props);
 228         }
 229
 230         logger.trace("handlePortStatusMessage {} type {}", nodeConnector, type);
 231
 232         if (type != null) {
 233             notifyInventoryShimListener(nodeConnector, type, props);
 234         }
 235     }
 236
 237     @Override
 238     public void switchAdded(ISwitch sw) {
 239         if (sw == null) {
 240             return;
 241         }
 242
 243         // Add all the nodeConnectors of this switch
 244         Map<NodeConnector, Set<Property>> ncProps = InventoryServiceHelper
 245                 .OFSwitchToProps(sw);
 246         for (Map.Entry<NodeConnector, Set<Property>> entry : ncProps.entrySet()) {
 247             Set<Property> props = new HashSet<Property>();
 248             Set<Property> prop = entry.getValue();
 249             if (prop != null) {
 250                 props.addAll(prop);
 251             }
 252             nodeConnectorProps.put(entry.getKey(), props);
 253             notifyInventoryShimListener(entry.getKey(), UpdateType.ADDED,
 254                     entry.getValue());
 255         }
 256
 257         // Add this node
 258         addNode(sw);
 259     }
 260
 261     @Override
 262     public void switchDeleted(ISwitch sw) {
 263         if (sw == null) {
 264             return;
 265         }
 266
 267         removeNode(sw);
 268     }
 269
 270     @Override
 271     public void containerModeUpdated(UpdateType t) {
 272         // do nothing
 273     }
 274
 275     @Override
 276     public void tagUpdated(String containerName, Node n, short oldTag,
 277             short newTag, UpdateType t) {
 278         logger.debug("tagUpdated: {} type {} for container {}", new Object[] {
 279                 n, t, containerName });
 280     }
 281
 282     @Override
 283     public void containerFlowUpdated(String containerName,
 284             ContainerFlow previousFlow, ContainerFlow currentFlow, UpdateType t) {
 285     }
 286
 287     @Override
 288     public void nodeConnectorUpdated(String containerName, NodeConnector nc, UpdateType t) {
 289         logger.debug("nodeConnectorUpdated: {} type {} for container {}", new Object[] { nc, t, containerName });
 290         Node node = nc.getNode();
 291         Set<String> ncContainers = this.nodeConnectorContainerMap.get(nc);
 292         Set<String> nodeContainers = this.nodeContainerMap.get(node);
 293         if (ncContainers == null) {
 294             ncContainers = new CopyOnWriteArraySet<String>();
 295         }
 296         if (nodeContainers == null) {
 297             nodeContainers = new CopyOnWriteArraySet<String>();
 298         }
 299         boolean notifyNodeUpdate = false;
 300
 301         switch (t) {
 302         case ADDED:
 303             if (ncContainers.add(containerName)) {
 304                 this.nodeConnectorContainerMap.put(nc, ncContainers);
 305             }
 306             if (nodeContainers.add(containerName)) {
 307                 this.nodeContainerMap.put(node, nodeContainers);
 308                 notifyNodeUpdate = true;
 309             }
 310             break;
 311         case REMOVED:
 312             if (ncContainers.remove(containerName)) {
 313                 if (ncContainers.isEmpty()) {
 314                     // Do cleanup to reduce memory footprint if no
 315                     // elements to be tracked
 316                     this.nodeConnectorContainerMap.remove(nc);
 317                 } else {
 318                     this.nodeConnectorContainerMap.put(nc, ncContainers);
 319                 }
 320             }
 321             boolean nodeContainerUpdate = true;
 322             for (NodeConnector ncContainer : nodeConnectorContainerMap.keySet()) {
 323                 if ((ncContainer.getNode().equals(node)) && (nodeConnectorContainerMap.get(ncContainer).contains(containerName))) {
 324                     nodeContainerUpdate = false;
 325                     break;
 326                 }
 327             }
 328             if (nodeContainerUpdate) {
 329                 nodeContainers.remove(containerName);
 330                 notifyNodeUpdate = true;
 331                 if (nodeContainers.isEmpty()) {
 332                     this.nodeContainerMap.remove(node);
 333                 } else {
 334                     this.nodeContainerMap.put(node, nodeContainers);
 335                 }
 336             }
 337             break;
 338         case CHANGED:
 339             break;
 340         }
 341
 342         Set<Property> nodeProp = nodeProps.get(node);
 343         if (nodeProp == null) {
 344             return;
 345         }
 346         Set<Property> ncProp = nodeConnectorProps.get(nc);
 347         // notify InventoryService
 348         notifyInventoryShimInternalListener(containerName, nc, t, ncProp);
 349
 350         if (notifyNodeUpdate) {
 351             notifyInventoryShimInternalListener(containerName, node, t, nodeProp);
 352         }
 353     }
 354
 355     private void notifyInventoryShimExternalListener(Node node,
 356             UpdateType type, Set<Property> props) {
 357         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
 358             s.updateNode(node, type, props);
 359         }
 360     }
 361
 362     private void notifyInventoryShimExternalListener(
 363             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
 364         for (IInventoryShimExternalListener s : this.inventoryShimExternalListeners) {
 365             s.updateNodeConnector(nodeConnector, type, props);
 366         }
 367     }
 368
 369     private void notifyInventoryShimInternalListener(String container,
 370             NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
 371         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
 372                 .get(container);
 373         if (inventoryShimInternalListener != null) {
 374             inventoryShimInternalListener.updateNodeConnector(nodeConnector,
 375                     type, props);
 376             logger.trace(
 377                     "notifyInventoryShimInternalListener {} type {} for container {}",
 378                     new Object[] { nodeConnector, type, container });
 379         }
 380     }
 381
 382     /*
 383      * Notify all internal and external listeners
 384      */
 385     private void notifyInventoryShimListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
 386
 387         //establish locality before notifying
 388         boolean isNodeLocal;
 389         if (type == UpdateType.REMOVED){
 390             //if removing get the locality first
 391             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
 392             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
 393         } else {
 394             notifyGlobalInventoryShimInternalListener(nodeConnector, type, props);
 395             isNodeLocal = connectionOutService.isLocal(nodeConnector.getNode());
 396         }
 397
 398         if (isNodeLocal) {
 399             // notify other containers
 400             Set<String> containers = (nodeConnectorContainerMap.get(nodeConnector) == null) ? new HashSet<String>()
 401                     : new HashSet<String>(nodeConnectorContainerMap.get(nodeConnector));
 402             containers.add(GlobalConstants.DEFAULT.toString());
 403             for (String container : containers) {
 404                 notifyInventoryShimInternalListener(container, nodeConnector, type, props);
 405             }
 406
 407             // Notify DiscoveryService
 408             notifyInventoryShimExternalListener(nodeConnector, type, props);
 409
 410             logger.debug("Connection service accepted the inventory notification for {} {}", nodeConnector, type);
 411         } else {
 412             logger.debug("Connection service dropped the inventory notification for {} {}", nodeConnector, type);
 413         }
 414     }
 415
 416     /*
 417      * Notify all internal and external listeners
 418      */
 419     private void notifyInventoryShimListener(Node node, UpdateType type, Set<Property> props) {
 420
 421         //establish locality before notifying
 422         boolean isNodeLocal;
 423         if (type == UpdateType.REMOVED){
 424             //if removing get the locality first
 425             isNodeLocal = connectionOutService.isLocal(node);
 426             notifyGlobalInventoryShimInternalListener(node, type, props);
 427         } else {
 428             notifyGlobalInventoryShimInternalListener(node, type, props);
 429             isNodeLocal = connectionOutService.isLocal(node);
 430         }
 431
 432         if (isNodeLocal) {
 433             // Now notify other containers
 434             Set<String> containers = (nodeContainerMap.get(node) == null) ? new HashSet<String>() : new HashSet<String>(
 435                     nodeContainerMap.get(node));
 436             containers.add(GlobalConstants.DEFAULT.toString());
 437             for (String container : containers) {
 438                 notifyInventoryShimInternalListener(container, node, type, props);
 439             }
 440
 441             // Notify external listener
 442             notifyInventoryShimExternalListener(node, type, props);
 443
 444             logger.debug("Connection service accepted the inventory notification for {} {}", node, type);
 445         } else {
 446             logger.debug("Connection service dropped the inventory notification for {} {}", node, type);
 447         }
 448     }
 449
 450     private void notifyGlobalInventoryShimInternalListener(Node node, UpdateType type, Set<Property> props) {
 451         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
 452             globalListener.updateNode(node, type, props);
 453             logger.trace(
 454                     "notifyGlobalInventoryShimInternalListener {} type {}",
 455                     new Object[] { node, type });
 456         }
 457     }
 458
 459     private void notifyGlobalInventoryShimInternalListener(NodeConnector nodeConnector, UpdateType type, Set<Property> props) {
 460         for (IInventoryShimInternalListener globalListener : globalInventoryShimInternalListeners) {
 461             globalListener.updateNodeConnector(nodeConnector, type, props);
 462             logger.trace(
 463                     "notifyGlobalInventoryShimInternalListener {} type {}",
 464                     new Object[] { nodeConnector, type });
 465         }
 466     }
 467
 468     private void notifyInventoryShimInternalListener(String container,
 469             Node node, UpdateType type, Set<Property> props) {
 470         IInventoryShimInternalListener inventoryShimInternalListener = inventoryShimInternalListeners
 471                 .get(container);
 472         if (inventoryShimInternalListener != null) {
 473             inventoryShimInternalListener.updateNode(node, type, props);
 474             logger.trace(
 475                     "notifyInventoryShimInternalListener {} type {} for container {}",
 476                     new Object[] { node, type, container });
 477         }
 478     }
 479
 480     private void addNode(ISwitch sw) {
 481         Node node = NodeCreator.createOFNode(sw.getId());
 482         UpdateType type = UpdateType.ADDED;
 483
 484         Set<Property> props = new HashSet<Property>();
 485         Long sid = (Long) node.getID();
 486
 487         Date connectedSince = controller.getSwitches().get(sid)
 488                 .getConnectedDate();
 489         Long connectedSinceTime = (connectedSince == null) ? 0 : connectedSince
 490                 .getTime();
 491         props.add(new TimeStamp(connectedSinceTime, "connectedSince"));
 492         props.add(new MacAddress(deriveMacAddress(sid)));
 493
 494         byte tables = sw.getTables();
 495         Tables t = new Tables(tables);
 496         if (t != null) {
 497             props.add(t);
 498         }
 499         int cap = sw.getCapabilities();
 500         Capabilities c = new Capabilities(cap);
 501         if (c != null) {
 502             props.add(c);
 503         }
 504         int act = sw.getActions();
 505         Actions a = new Actions(act);
 506         if (a != null) {
 507             props.add(a);
 508         }
 509         int buffers = sw.getBuffers();
 510         Buffers b = new Buffers(buffers);
 511         if (b != null) {
 512             props.add(b);
 513         }
 514
 515         nodeProps.put(node, props);
 516         // Notify all internal and external listeners
 517         notifyInventoryShimListener(node, type, props);
 518     }
 519
 520     private void removeNode(ISwitch sw) {
 521         Node node = NodeCreator.createOFNode(sw.getId());
 522         if(node == null) {
 523             return;
 524         }
 525         removeNodeConnectorProps(node);
 526         nodeProps.remove(node);
 527         UpdateType type = UpdateType.REMOVED;
 528         // Notify all internal and external listeners
 529         notifyInventoryShimListener(node, type, null);
 530     }
 531
 532     private void startService() {
 533         // Get a snapshot of all the existing switches
 534         Map<Long, ISwitch> switches = this.controller.getSwitches();
 535         for (ISwitch sw : switches.values()) {
 536             switchAdded(sw);
 537         }
 538     }
 539
 540     private void removeNodeConnectorProps(Node node) {
 541         List<NodeConnector> ncList = new ArrayList<NodeConnector>();
 542         for (NodeConnector nc : nodeConnectorProps.keySet()) {
 543             if (nc.getNode().equals(node)) {
 544                 ncList.add(nc);
 545             }
 546         }
 547         for (NodeConnector nc : ncList) {
 548             nodeConnectorProps.remove(nc);
 549         }
 550     }
 551
 552     @Override
 553     public void descriptionStatisticsRefreshed(Long switchId, List<OFStatistics> descriptionStats) {
 554         Node node = NodeCreator.createOFNode(switchId);
 555         Set<Property> properties = new HashSet<Property>(1);
 556         OFDescriptionStatistics ofDesc = (OFDescriptionStatistics) descriptionStats.get(0);
 557         Description desc = new Description(ofDesc.getDatapathDescription());
 558         properties.add(desc);
 559
 560         // Notify all internal and external listeners
 561         notifyInventoryShimListener(node, UpdateType.CHANGED, properties);
 562     }
 563
 564     private byte[] deriveMacAddress(long dpid) {
 565         byte[] mac = new byte[] { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
 566
 567         for (short i = 0; i < 6; i++) {
 568             mac[5 - i] = (byte) dpid;
 569             dpid >>= 8;
 570         }
 571
 572         return mac;
 573     }
 574
 575     @Override
 576     public void flowStatisticsRefreshed(Long switchId, List<OFStatistics> flows) {
 577         // Nothing to do
 578     }
 579
 580     @Override
 581     public void portStatisticsRefreshed(Long switchId, List<OFStatistics> ports) {
 582         // Nothing to do
 583     }
 584
 585     @Override
 586     public void tableStatisticsRefreshed(Long switchId, List<OFStatistics> tables) {
 587         // Nothing to do
 588     }
 589
 590     @Override
 591     public void containerCreate(String containerName) {
 592         // Nothing to do
 593     }
 594
 595     @Override
 596     public void containerDestroy(String containerName) {
 597         Set<NodeConnector> removeNodeConnectorSet = new HashSet<NodeConnector>();
 598         Set<Node> removeNodeSet = new HashSet<Node>();
 599         for (Map.Entry<NodeConnector, Set<String>> entry : nodeConnectorContainerMap.entrySet()) {
 600             Set<String> ncContainers = entry.getValue();
 601             if (ncContainers.contains(containerName)) {
 602                 NodeConnector nodeConnector = entry.getKey();
 603                 removeNodeConnectorSet.add(nodeConnector);
 604             }
 605         }
 606         for (Map.Entry<Node, Set<String>> entry : nodeContainerMap.entrySet()) {
 607             Set<String> nodeContainers = entry.getValue();
 608             if (nodeContainers.contains(containerName)) {
 609                 Node node = entry.getKey();
 610                 removeNodeSet.add(node);
 611             }
 612         }
 613         for (NodeConnector nodeConnector : removeNodeConnectorSet) {
 614             Set<String> ncContainers = nodeConnectorContainerMap.get(nodeConnector);
 615             ncContainers.remove(containerName);
 616             if (ncContainers.isEmpty()) {
 617                 nodeConnectorContainerMap.remove(nodeConnector);
 618             }
 619         }
 620         for (Node node : removeNodeSet) {
 621             Set<String> nodeContainers = nodeContainerMap.get(node);
 622             nodeContainers.remove(containerName);
 623             if (nodeContainers.isEmpty()) {
 624                 nodeContainerMap.remove(node);
 625             }
 626         }
 627     }
 628 }