neutron/src/main/java/org/opendaylight/ovsdb/neutron/provider/OF10ProviderManager.java

   1 package org.opendaylight.ovsdb.neutron.provider;
   2
   3 import java.math.BigInteger;
   4 import java.net.InetAddress;
   5 import java.util.ArrayList;
   6 import java.util.HashMap;
   7 import java.util.List;
   8 import java.util.Map;
   9 import java.util.Set;
  10
  11 import org.opendaylight.controller.forwardingrulesmanager.FlowConfig;
  12 import org.opendaylight.controller.forwardingrulesmanager.IForwardingRulesManager;
  13 import org.opendaylight.controller.sal.action.ActionType;
  14 import org.opendaylight.controller.sal.core.Node;
  15 import org.opendaylight.controller.sal.utils.HexEncode;
  16 import org.opendaylight.controller.sal.utils.ServiceHelper;
  17 import org.opendaylight.controller.sal.utils.Status;
  18 import org.opendaylight.controller.sal.utils.StatusCode;
  19 import org.opendaylight.ovsdb.lib.notation.OvsDBMap;
  20 import org.opendaylight.ovsdb.lib.notation.OvsDBSet;
  21 import org.opendaylight.ovsdb.lib.notation.UUID;
  22 import org.opendaylight.ovsdb.lib.table.Bridge;
  23 import org.opendaylight.ovsdb.lib.table.Interface;
  24 import org.opendaylight.ovsdb.lib.table.Port;
  25 import org.opendaylight.ovsdb.lib.table.internal.Table;
  26 import org.opendaylight.ovsdb.neutron.AdminConfigManager;
  27 import org.opendaylight.ovsdb.neutron.InternalNetworkManager;
  28 import org.opendaylight.ovsdb.neutron.TenantNetworkManager;
  29 import org.opendaylight.ovsdb.plugin.IConnectionServiceInternal;
  30 import org.opendaylight.ovsdb.plugin.OVSDBConfigService;
  31 import org.opendaylight.ovsdb.plugin.StatusWithUuid;
  32 import org.slf4j.Logger;
  33 import org.slf4j.LoggerFactory;
  34
  35
  36 class OF10ProviderManager extends ProviderNetworkManager {
  37     private static final Logger logger = LoggerFactory.getLogger(OF10ProviderManager.class);
  38     Map<NodeVlan, FlowConfig> floodEntries = new HashMap<NodeVlan, FlowConfig>();
  39     private static final int INGRESS_TUNNEL_FLOW_PRIORITY = 100;
  40     private static final int EGRESS_TUNNEL_FLOW_PRIORITY = 100;
  41     private static final int FLOOD_TUNNEL_FLOW_PRIORITY = 1;
  42
  43     @Override
  44     public boolean hasPerTenantTunneling() {
  45         return true;
  46     }
  47
  48     private Status getTunnelReadinessStatus (Node node, String tunnelKey) {
  49         InetAddress srcTunnelEndPoint = AdminConfigManager.getManager().getTunnelEndPoint(node);
  50         if (srcTunnelEndPoint == null) {
  51             logger.error("Tunnel Endpoint not configured for Node {}", node);
  52             return new Status(StatusCode.NOTFOUND, "Tunnel Endpoint not configured for "+ node);
  53         }
  54
  55         try {
  56             if (!InternalNetworkManager.getManager().isInternalNetworkOverlayReady(node)) {
  57                 logger.error(node+" is not Overlay ready");
  58                 return new Status(StatusCode.NOTACCEPTABLE, node+" is not Overlay ready");
  59             }
  60         } catch (Exception e) {
  61             logger.error(node+" is not Overlay ready due to exception", e);
  62             return new Status(StatusCode.NOTACCEPTABLE, node+" is not Overlay ready");
  63         }
  64
  65         if (!TenantNetworkManager.getManager().isTenantNetworkPresentInNode(node, tunnelKey)) {
  66             logger.debug(node+" has no VM corresponding to segment "+ tunnelKey);
  67             return new Status(StatusCode.NOTACCEPTABLE, node+" has no VM corresponding to segment "+ tunnelKey);
  68         }
  69         return new Status(StatusCode.SUCCESS);
  70     }
  71
  72     /**
  73      * Program OF1.0 Flow rules on br-tun on the ingress direction from the network towards the br-int.
  74      * The logic is to simply match on the incoming tunnel OF-Port (which carries the TenantNetwork GRE-Key)
  75      * and rewrite the Corresponding internal Vlan and pass it on to br-int via the patch port.
  76      */
  77     private void programLocalIngressTunnelBridgeRules(Node node, int tunnelOFPort, int internalVlan, int patchPort) {
  78         String brIntId = InternalNetworkManager.getManager().getInternalBridgeUUID(node, AdminConfigManager.getManager().getTunnelBridgeName());
  79         if (brIntId == null) {
  80             logger.error("Failed to initialize Flow Rules for {}", node);
  81             return;
  82         }
  83         try {
  84             String flowName = "TepMatch"+tunnelOFPort+""+internalVlan;
  85             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
  86             Bridge bridge = (Bridge) ovsdbTable.getRow(node, Bridge.NAME.getName(), brIntId);
  87             Set<String> dpids = bridge.getDatapath_id();
  88             if (dpids == null || dpids.size() ==  0) return;
  89             Long dpidLong = Long.valueOf(HexEncode.stringToLong((String)dpids.toArray()[0]));
  90             Node ofNode = new Node(Node.NodeIDType.OPENFLOW, dpidLong);
  91             IForwardingRulesManager frm = (IForwardingRulesManager) ServiceHelper.getInstance(
  92                     IForwardingRulesManager.class, "default", this);
  93             if (frm.getStaticFlow(flowName, ofNode) != null) {
  94                 logger.debug("Local Ingress Flow exists : {} for Flow {} on {} / {}", flowName, ofNode, node);
  95                 return;
  96             }
  97
  98             FlowConfig flow = new FlowConfig();
  99             flow.setName(flowName);
 100             flow.setNode(ofNode);
 101             flow.setPriority(INGRESS_TUNNEL_FLOW_PRIORITY+"");
 102             flow.setIngressPort(tunnelOFPort+"");
 103             List<String> actions = new ArrayList<String>();
 104             actions.add(ActionType.SET_VLAN_ID+"="+internalVlan);
 105             actions.add(ActionType.OUTPUT.toString()+"="+patchPort);
 106             flow.setActions(actions);
 107             Status status = frm.addStaticFlow(flow);
 108             logger.debug("Local Ingress Flow Programming Status {} for Flow {} on {} / {}", status, flow, ofNode, node);
 109         } catch (Exception e) {
 110             logger.error("Failed to initialize Flow Rules for {}", node, e);
 111         }
 112     }
 113
 114     /**
 115      * Program OF1.0 Flow rules on br-tun on the remote Node on its egress direction towards the overlay network
 116      * for a VM (with the attachedMac).
 117      * The logic is to simply match on the incoming vlan, mac from the patch-port connected to br-int (patch-int)
 118      * and output the traffic to the appropriate GRE Tunnel (which carries the GRE-Key for that Tenant Network).
 119      * Also perform the Strip-Vlan action.
 120      */
 121     private void programRemoteEgressTunnelBridgeRules(Node node, int patchPort, String attachedMac,
 122             int internalVlan, int tunnelOFPort) {
 123         String brIntId = InternalNetworkManager.getManager().getInternalBridgeUUID(node, AdminConfigManager.getManager().getTunnelBridgeName());
 124         if (brIntId == null) {
 125             logger.error("Failed to initialize Flow Rules for {}", node);
 126             return;
 127         }
 128         try {
 129             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 130             Bridge bridge = (Bridge) ovsdbTable.getRow(node, Bridge.NAME.getName(), brIntId);
 131             Set<String> dpids = bridge.getDatapath_id();
 132             if (dpids == null || dpids.size() ==  0) return;
 133             Long dpidLong = Long.valueOf(HexEncode.stringToLong((String)dpids.toArray()[0]));
 134             Node ofNode = new Node(Node.NodeIDType.OPENFLOW, dpidLong);
 135             String flowName = "TepMatch"+tunnelOFPort+""+internalVlan+""+HexEncode.stringToLong(attachedMac);
 136             IForwardingRulesManager frm = (IForwardingRulesManager) ServiceHelper.getInstance(
 137                     IForwardingRulesManager.class, "default", this);
 138             if (frm.getStaticFlow(flowName, ofNode) != null) {
 139                 logger.debug("Remote Egress Flow exists : {} for Flow {} on {} / {}", flowName, ofNode, node);
 140                 return;
 141             }
 142             FlowConfig flow = new FlowConfig();
 143             flow.setName(flowName);
 144             flow.setNode(ofNode);
 145             flow.setPriority(EGRESS_TUNNEL_FLOW_PRIORITY+"");
 146             flow.setDstMac(attachedMac);
 147             flow.setIngressPort(patchPort+"");
 148             flow.setVlanId(internalVlan+"");
 149             List<String> actions = new ArrayList<String>();
 150             actions.add(ActionType.POP_VLAN.toString());
 151             actions.add(ActionType.OUTPUT.toString()+"="+tunnelOFPort);
 152             flow.setActions(actions);
 153             Status status = frm.addStaticFlow(flow);
 154             logger.debug("Remote Egress Flow Programming Status {} for Flow {} on {} / {}", status, flow, ofNode, node);
 155         } catch (Exception e) {
 156             logger.error("Failed to initialize Flow Rules for {}", node, e);
 157         }
 158     }
 159
 160     /**
 161      * Program OF1.0 Flow rules to flood the broadcast & unknown-unicast traffic over br-tun on the egress direction
 162      * towards the network on all the overlay tunnels that corresponds to the tenant network.
 163      * The logic is to simply match on the incoming vlan, mac from the patch-port connected to br-int (patch-int)
 164      * and output the traffic to all the GRE-Tunnels for this Tenant Network (which carries the GRE-Key).
 165      * Also perform the Strip-Vlan action.
 166      */
 167     private void programFloodEgressTunnelBridgeRules(Node node, int patchPort, int internalVlan, int tunnelOFPort) {
 168         String brIntId = InternalNetworkManager.getManager().getInternalBridgeUUID(node, AdminConfigManager.getManager().getTunnelBridgeName());
 169         if (brIntId == null) {
 170             logger.error("Failed to initialize Flow Rules for {}", node);
 171             return;
 172         }
 173         try {
 174             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 175             Bridge bridge = (Bridge) ovsdbTable.getRow(node, Bridge.NAME.getName(), brIntId);
 176             Set<String> dpids = bridge.getDatapath_id();
 177             if (dpids == null || dpids.size() ==  0) return;
 178             Long dpidLong = Long.valueOf(HexEncode.stringToLong((String)dpids.toArray()[0]));
 179             Node ofNode = new Node(Node.NodeIDType.OPENFLOW, dpidLong);
 180             NodeVlan nv = new NodeVlan(ofNode, internalVlan);
 181             FlowConfig existingFlowConfig = floodEntries.get(nv);
 182             IForwardingRulesManager frm = (IForwardingRulesManager) ServiceHelper.getInstance(
 183                     IForwardingRulesManager.class, "default", this);
 184             FlowConfig flow = existingFlowConfig;
 185             Status status = null;
 186             if (flow == null) {
 187                 flow = new FlowConfig();
 188                 flow.setName("TepFlood"+internalVlan);
 189                 flow.setNode(ofNode);
 190                 flow.setPriority(FLOOD_TUNNEL_FLOW_PRIORITY+"");
 191                 flow.setIngressPort(patchPort+"");
 192                 flow.setVlanId(internalVlan+"");
 193                 List<String> actions = new ArrayList<String>();
 194                 actions.add(ActionType.POP_VLAN.toString());
 195                 actions.add(ActionType.OUTPUT.toString()+"="+tunnelOFPort);
 196                 flow.setActions(actions);
 197                 status = frm.addStaticFlow(flow);
 198                 logger.debug("Add Flood Egress Flow Programming Status {} for Flow {} on {} / {}",
 199                               status, flow, ofNode, node);
 200             } else {
 201                 flow = new FlowConfig(existingFlowConfig);
 202                 List<String> actions = flow.getActions();
 203                 String outputPort = ActionType.OUTPUT.toString()+"="+tunnelOFPort;
 204                 if (actions != null && !actions.contains(outputPort)) {
 205                     actions.add(outputPort);
 206                     flow.setActions(actions);
 207                 } else {
 208                     return;
 209                 }
 210                 status = frm.modifyStaticFlow(flow);
 211                 logger.debug("Modify Flood Egress Flow Programming Status {} for Flow {} on {} / {}",
 212                               status, flow, ofNode, node);
 213             }
 214             if (status.isSuccess()) {
 215                 floodEntries.put(nv, flow);
 216             }
 217
 218         } catch (Exception e) {
 219             logger.error("Failed to initialize Flow Rules for {}", node, e);
 220         }
 221     }
 222
 223     private void programTunnelRules (String tunnelType, String segmentationId, InetAddress dst, Node node,
 224                                      Interface intf, boolean local) {
 225         String networkId = TenantNetworkManager.getManager().getNetworkIdForSegmentationId(segmentationId);
 226         if (networkId == null) {
 227             logger.debug("Tenant Network not found with Segmenation-id {}",segmentationId);
 228             return;
 229         }
 230         int internalVlan = TenantNetworkManager.getManager().getInternalVlan(networkId);
 231         if (internalVlan == 0) {
 232             logger.debug("No InternalVlan provisioned for Tenant Network {}",networkId);
 233             return;
 234         }
 235         Map<String, String> externalIds = intf.getExternal_ids();
 236         if (externalIds == null) {
 237             logger.error("No external_ids seen in {}", intf);
 238             return;
 239         }
 240
 241         String attachedMac = externalIds.get(TenantNetworkManager.EXTERNAL_ID_VM_MAC);
 242         if (attachedMac == null) {
 243             logger.error("No AttachedMac seen in {}", intf);
 244             return;
 245         }
 246         String patchInt = AdminConfigManager.getManager().getPatchToIntegration();
 247
 248         int patchOFPort = -1;
 249         try {
 250             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 251             Map<String, Table<?>> intfs = ovsdbTable.getRows(node, Interface.NAME.getName());
 252             if (intfs != null) {
 253                 for (Table<?> row : intfs.values()) {
 254                     Interface patchIntf = (Interface)row;
 255                     if (patchIntf.getName().equalsIgnoreCase(patchInt)) {
 256                         Set<BigInteger> of_ports = patchIntf.getOfport();
 257                         if (of_ports == null || of_ports.size() <= 0) {
 258                             logger.error("Could NOT Identified Patch port {} on {}", patchInt, node);
 259                             continue;
 260                         }
 261                         patchOFPort = Long.valueOf(((BigInteger)of_ports.toArray()[0]).longValue()).intValue();
 262                         logger.debug("Identified Patch port {} -> OF ({}) on {}", patchInt, patchOFPort, node);
 263                         break;
 264                     }
 265                 }
 266                 if (patchOFPort == -1) {
 267                     logger.error("Cannot identify {} interface on {}", patchInt, node);
 268                 }
 269                 for (Table<?> row : intfs.values()) {
 270                     Interface tunIntf = (Interface)row;
 271                     if (tunIntf.getName().equals(this.getTunnelName(tunnelType, segmentationId, dst))) {
 272                         Set<BigInteger> of_ports = tunIntf.getOfport();
 273                         if (of_ports == null || of_ports.size() <= 0) {
 274                             logger.error("Could NOT Identify Tunnel port {} on {}", tunIntf.getName(), node);
 275                             continue;
 276                         }
 277                         int tunnelOFPort = Long.valueOf(((BigInteger)of_ports.toArray()[0]).longValue()).intValue();
 278
 279                         if (tunnelOFPort == -1) {
 280                             logger.error("Could NOT Identify Tunnel port {} -> OF ({}) on {}", tunIntf.getName(), tunnelOFPort, node);
 281                             return;
 282                         }
 283                         logger.debug("Identified Tunnel port {} -> OF ({}) on {}", tunIntf.getName(), tunnelOFPort, node);
 284
 285                         if (!local) {
 286                             programRemoteEgressTunnelBridgeRules(node, patchOFPort, attachedMac, internalVlan, tunnelOFPort);
 287                         }
 288                         programLocalIngressTunnelBridgeRules(node, tunnelOFPort, internalVlan, patchOFPort);
 289                         programFloodEgressTunnelBridgeRules(node, patchOFPort, internalVlan, tunnelOFPort);
 290                         return;
 291                     }
 292                 }
 293             }
 294         } catch (Exception e) {
 295             logger.error("", e);
 296         }
 297     }
 298
 299     @Override
 300     public Status createTunnels(String tunnelType, String tunnelKey, Node srcNode, Interface intf) {
 301         Status status = getTunnelReadinessStatus(srcNode, tunnelKey);
 302         if (!status.isSuccess()) return status;
 303
 304         IConnectionServiceInternal connectionService = (IConnectionServiceInternal)ServiceHelper.getGlobalInstance(IConnectionServiceInternal.class, this);
 305         List<Node> nodes = connectionService.getNodes();
 306         nodes.remove(srcNode);
 307         for (Node dstNode : nodes) {
 308             status = getTunnelReadinessStatus(dstNode, tunnelKey);
 309             if (!status.isSuccess()) continue;
 310             InetAddress src = AdminConfigManager.getManager().getTunnelEndPoint(srcNode);
 311             InetAddress dst = AdminConfigManager.getManager().getTunnelEndPoint(dstNode);
 312             status = addTunnelPort(srcNode, tunnelType, src, dst, tunnelKey);
 313             if (status.isSuccess()) {
 314                 this.programTunnelRules(tunnelType, tunnelKey, dst, srcNode, intf, true);
 315             }
 316             addTunnelPort(dstNode, tunnelType, dst, src, tunnelKey);
 317             if (status.isSuccess()) {
 318                 this.programTunnelRules(tunnelType, tunnelKey, src, dstNode, intf, false);
 319             }
 320         }
 321         return new Status(StatusCode.SUCCESS);
 322     }
 323
 324     private String getTunnelName(String tunnelType, String key, InetAddress dst) {
 325         return tunnelType+"-"+key+"-"+dst.getHostAddress();
 326     }
 327
 328     private Interface getTunnelInterface (Node node, String tunnelType, InetAddress dst, String key) {
 329         try {
 330             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 331             String portName = getTunnelName(tunnelType, key, dst);
 332
 333             Map<String, Table<?>> tunIntfs = ovsdbTable.getRows(node, Interface.NAME.getName());
 334             if (tunIntfs != null) {
 335                 for (Table<?> row : tunIntfs.values()) {
 336                     Interface tunIntf = (Interface)row;
 337                     if (tunIntf.getName().equals(portName)) return tunIntf;
 338                 }
 339
 340             }
 341         } catch (Exception e) {
 342             logger.error("", e);
 343         }
 344         return null;
 345     }
 346
 347     private boolean isTunnelPresent(Node node, String tunnelName, String bridgeUUID) throws Exception {
 348         OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 349         Bridge bridge = (Bridge)ovsdbTable.getRow(node, Bridge.NAME.getName(), bridgeUUID);
 350         if (bridge != null) {
 351             Set<UUID> ports = bridge.getPorts();
 352             for (UUID portUUID : ports) {
 353                 Port port = (Port)ovsdbTable.getRow(node, Port.NAME.getName(), portUUID.toString());
 354                 if (port != null && port.getName().equalsIgnoreCase(tunnelName)) return true;
 355             }
 356         }
 357         return false;
 358     }
 359
 360     private Status addTunnelPort (Node node, String tunnelType, InetAddress src, InetAddress dst, String key) {
 361         try {
 362             String bridgeUUID = null;
 363             String tunnelBridgeName = AdminConfigManager.getManager().getTunnelBridgeName();
 364             OVSDBConfigService ovsdbTable = (OVSDBConfigService)ServiceHelper.getGlobalInstance(OVSDBConfigService.class, this);
 365             Map<String, Table<?>> bridgeTable = ovsdbTable.getRows(node, Bridge.NAME.getName());
 366             if (bridgeTable != null) {
 367                 for (String uuid : bridgeTable.keySet()) {
 368                     Bridge bridge = (Bridge)bridgeTable.get(uuid);
 369                     if (bridge.getName().equals(tunnelBridgeName)) {
 370                         bridgeUUID = uuid;
 371                         break;
 372                     }
 373                 }
 374             }
 375             if (bridgeUUID == null) {
 376                 logger.error("Could not find Bridge {} in {}", tunnelBridgeName, node);
 377                 return new Status(StatusCode.NOTFOUND, "Could not find "+tunnelBridgeName+" in "+node);
 378             }
 379             String portName = getTunnelName(tunnelType, key, dst);
 380
 381             if (this.isTunnelPresent(node, portName, bridgeUUID)) {
 382                 logger.trace("Tunnel {} is present in {} of {}", portName, tunnelBridgeName, node);
 383                 return new Status(StatusCode.SUCCESS);
 384             }
 385
 386             Port tunnelPort = new Port();
 387             tunnelPort.setName(portName);
 388             StatusWithUuid statusWithUuid = ovsdbTable.insertRow(node, Port.NAME.getName(), bridgeUUID, tunnelPort);
 389             if (!statusWithUuid.isSuccess()) {
 390                 logger.error("Failed to insert Tunnel port {} in {}", portName, bridgeUUID);
 391                 return statusWithUuid;
 392             }
 393
 394             String tunnelPortUUID = statusWithUuid.getUuid().toString();
 395             String interfaceUUID = null;
 396             int timeout = 6;
 397             while ((interfaceUUID == null) && (timeout > 0)) {
 398                 tunnelPort = (Port)ovsdbTable.getRow(node, Port.NAME.getName(), tunnelPortUUID);
 399                 OvsDBSet<UUID> interfaces = tunnelPort.getInterfaces();
 400                 if (interfaces == null || interfaces.size() == 0) {
 401                     // Wait for the OVSDB update to sync up the Local cache.
 402                     Thread.sleep(500);
 403                     timeout--;
 404                     continue;
 405                 }
 406                 interfaceUUID = interfaces.toArray()[0].toString();
 407                 Interface intf = (Interface)ovsdbTable.getRow(node, Interface.NAME.getName(), interfaceUUID);
 408                 if (intf == null) interfaceUUID = null;
 409             }
 410
 411             if (interfaceUUID == null) {
 412                 logger.error("Cannot identify Tunnel Interface for port {}/{}", portName, tunnelPortUUID);
 413                 return new Status(StatusCode.INTERNALERROR);
 414             }
 415
 416             Interface tunInterface = new Interface();
 417             tunInterface.setType(tunnelType);
 418             OvsDBMap<String, String> options = new OvsDBMap<String, String>();
 419             options.put("key", key);
 420             options.put("local_ip", src.getHostAddress());
 421             options.put("remote_ip", dst.getHostAddress());
 422             tunInterface.setOptions(options);
 423             Status status = ovsdbTable.updateRow(node, Interface.NAME.getName(), tunnelPortUUID, interfaceUUID, tunInterface);
 424             logger.debug("Tunnel {} add status : {}", tunInterface, status);
 425             return status;
 426         } catch (Exception e) {
 427             logger.error("Exception in addTunnelPort", e);
 428             return new Status(StatusCode.INTERNALERROR);
 429         }
 430     }
 431
 432     @Override
 433     public Status createTunnels(String tunnelType, String tunnelKey) {
 434         IConnectionServiceInternal connectionService = (IConnectionServiceInternal)ServiceHelper.getGlobalInstance(IConnectionServiceInternal.class, this);
 435         List<Node> nodes = connectionService.getNodes();
 436         for (Node srcNode : nodes) {
 437             this.createTunnels(tunnelType, tunnelKey, srcNode, null);
 438         }
 439         return new Status(StatusCode.SUCCESS);
 440     }
 441
 442     private class NodeVlan {
 443         Node node;
 444         int vlan;
 445         public NodeVlan(Node node, int vlan) {
 446             super();
 447             this.node = node;
 448             this.vlan = vlan;
 449         }
 450         public Node getNode() {
 451             return node;
 452         }
 453         public int getVlan() {
 454             return vlan;
 455         }
 456         @Override
 457         public String toString() {
 458             return "NodeVlan [node=" + node + ", vlan=" + vlan + "]";
 459         }
 460         @Override
 461         public int hashCode() {
 462             final int prime = 31;
 463             int result = 1;
 464             result = prime * result + ((node == null) ? 0 : node.hashCode());
 465             result = prime * result + vlan;
 466             return result;
 467         }
 468         @Override
 469         public boolean equals(Object obj) {
 470             if (this == obj)
 471                 return true;
 472             if (obj == null)
 473                 return false;
 474             if (getClass() != obj.getClass())
 475                 return false;
 476             NodeVlan other = (NodeVlan) obj;
 477             if (node == null) {
 478                 if (other.node != null)
 479                     return false;
 480             } else if (!node.equals(other.node))
 481                 return false;
 482             if (vlan != other.vlan)
 483                 return false;
 484             return true;
 485         }
 486     }
 487 }