vpnservice/dhcpservice/dhcpservice-impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import java.math.BigInteger;
  11 import java.util.ArrayList;
  12 import java.util.Collection;
  13 import java.util.HashSet;
  14 import java.util.LinkedList;
  15 import java.util.List;
  16 import java.util.Set;
  17 import java.util.concurrent.ConcurrentHashMap;
  18 import java.util.concurrent.ConcurrentMap;
  19 import java.util.concurrent.ExecutionException;
  20 import java.util.concurrent.Future;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  26 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.netvirt.dhcpservice.api.DHCPMConstants;
  29 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  30 import org.opendaylight.netvirt.neutronvpn.api.l2gw.utils.L2GatewayCacheUtils;
  31 import org.opendaylight.genius.mdsalutil.MDSALDataStoreUtils;
  32 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  33 import org.opendaylight.genius.mdsalutil.NwConstants;
  34 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  35 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  36 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  37 import org.opendaylight.genius.utils.clustering.ClusteringUtils;
  38 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  39 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  40 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  41 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  42 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  43 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  44 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  45 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorAugmentation;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  67 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  68 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  69 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  70 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  71 import org.opendaylight.yangtools.yang.common.RpcResult;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 import com.google.common.base.Optional;
  76 import com.google.common.util.concurrent.FutureCallback;
  77 import com.google.common.util.concurrent.Futures;
  78 import com.google.common.util.concurrent.ListenableFuture;
  79
  80 public class DhcpExternalTunnelManager {
  81
  82     private static final Logger logger = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
  83     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
  84     private final DataBroker broker;
  85     private IMdsalApiManager mdsalUtil;
  86
  87     private ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache = new ConcurrentHashMap<>();
  88     private ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache = new ConcurrentHashMap<>();
  89     private ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
  90     private ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
  91     private ItmRpcService itmRpcService;
  92     private EntityOwnershipService entityOwnershipService;
  93
  94     public DhcpExternalTunnelManager(DataBroker broker, IMdsalApiManager mdsalUtil, ItmRpcService itmRpcService, EntityOwnershipService entityOwnershipService) {
  95         this.broker = broker;
  96         this.mdsalUtil = mdsalUtil;
  97         this.itmRpcService = itmRpcService;
  98         this.entityOwnershipService = entityOwnershipService;
  99         initilizeCaches();
 100     }
 101
 102     private void initilizeCaches() {
 103         logger.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 104         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 105         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 106         if (designatedSwitchForTunnelOptional.isPresent()) {
 107             List<DesignatedSwitchForTunnel> list = designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 108             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 109                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair = designatedDpnsToTunnelIpElanNameCache.get(designatedSwitchForTunnel.getDpId());
 110                 if (setOfTunnelIpElanNamePair == null) {
 111                     setOfTunnelIpElanNamePair = new HashSet<>();
 112                 }
 113                 Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(), designatedSwitchForTunnel.getElanInstanceName());
 114                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 115                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()), setOfTunnelIpElanNamePair);
 116             }
 117         }
 118         logger.trace("Loading vniMacAddressToPortCache");
 119         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 120         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 121         if (optionalPorts.isPresent()) {
 122             List<Port> list = optionalPorts.get().getPort();
 123             for (Port port : list) {
 124                 if(NeutronUtils.isPortVnicTypeNormal(port)) {
 125                     continue;
 126                 }
 127                 String macAddress = port.getMacAddress().getValue();
 128                 Uuid networkId = port.getNetworkId();
 129                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 130                 if (segmentationId == null) {
 131                     return;
 132                 }
 133                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 134             }
 135         }
 136
 137     }
 138
 139
 140     public BigInteger designateDpnId(IpAddress tunnelIp,
 141             String elanInstanceName, List<BigInteger> dpns) {
 142         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 143         if (designatedDpnId != null && !designatedDpnId.equals(DHCPMConstants.INVALID_DPID)) {
 144             logger.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp, elanInstanceName);
 145             return designatedDpnId;
 146         }
 147         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 148     }
 149
 150     public void installDhcpFlowsForVms(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns,
 151             BigInteger designatedDpnId, String vmMacAddress ) {
 152         logger.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp, elanInstanceName, designatedDpnId, vmMacAddress);
 153         synchronized (getTunnelIpDpnKey(tunnelIp, designatedDpnId)) {
 154             installDhcpEntries(designatedDpnId, vmMacAddress, entityOwnershipService);
 155             dpns.remove(designatedDpnId);
 156             for (BigInteger dpn : dpns) {
 157                 installDhcpDropAction(dpn, vmMacAddress, entityOwnershipService);
 158             }
 159             updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 160         }
 161     }
 162
 163     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress) {
 164         for (String vmMacAddress : listVmMacAddress) {
 165             installDhcpEntries(designatedDpnId, vmMacAddress);
 166         }
 167     }
 168
 169     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 170         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 171         removeFromLocalCache(elanInstanceName, vmMacAddress);
 172     }
 173
 174     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 175         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 176             return null;
 177         }
 178         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 179         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 180         if (designatedSwitchForTunnelOptional.isPresent()) {
 181             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 182         }
 183         return null;
 184     }
 185
 186     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 187         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 188         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 189         DesignatedSwitchForTunnel designatedSwitchForTunnel = new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue()).setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp).setKey(designatedSwitchForTunnelKey).build();
 190         logger.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName, dpnId);
 191         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 192         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 193     }
 194
 195     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 196         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 197         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 198         logger.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName, dpnId);
 199         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 200         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 201     }
 202
 203     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 204         List<String> vmMacs = getAllVmMacs();
 205         logger.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 206         for (String vmMacAddress : vmMacs) {
 207             installDhcpDropAction(dpId, vmMacAddress);
 208         }
 209     }
 210
 211     private List<String> getAllVmMacs() {
 212         List<String> vmMacs = new LinkedList<>();
 213         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 214         for (Set<String> list : listOfVmMacs) {
 215             vmMacs.addAll(list);
 216         }
 217         return vmMacs;
 218     }
 219
 220     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 221         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 222         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 223         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 224         if (tunnelIpElanNameSet == null) {
 225             tunnelIpElanNameSet = new HashSet<>();
 226         }
 227         tunnelIpElanNameSet.add(tunnelIpElanName);
 228         logger.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId, tunnelIpElanNameSet);
 229         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 230     }
 231
 232     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 233         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 234         Set<String> setOfExistingVmMacAddress;
 235         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 236         if (setOfExistingVmMacAddress == null) {
 237             setOfExistingVmMacAddress = new HashSet<>();
 238         }
 239         setOfExistingVmMacAddress.add(vmMacAddress);
 240         logger.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName, setOfExistingVmMacAddress);
 241         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 242         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 243     }
 244
 245     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 246         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<IpAddress, String>(tunnelIp, elanInstanceName);
 247         Set<String> listExistingVmMacAddress;
 248         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 249         if (null == listExistingVmMacAddress) {
 250             listExistingVmMacAddress = new HashSet<String>();
 251         }
 252         listExistingVmMacAddress.add(vmMacAddress);
 253         logger.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 254                 listExistingVmMacAddress);
 255         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 256     }
 257
 258     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 259         logger.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 260         try {
 261             Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 262             if (!dpnId.equals(DHCPMConstants.INVALID_DPID)) {
 263                 List<String> listOfVms = getAllVmMacs();
 264                 for (String vmMacAddress : listOfVms) {
 265                     unInstallDhcpEntries(dpnId, vmMacAddress);
 266                 }
 267             }
 268             if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 269                 logger.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 270                 return;
 271             }
 272             for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 273                 updateCacheAndInstallNewFlows(dpnId, listOfDpns, pair);
 274             }
 275         } catch (Exception e) {
 276             logger.error("Error in handleDesignatedDpnDown {}", e);
 277         }
 278     }
 279
 280     public void updateCacheAndInstallNewFlows(BigInteger dpnId,
 281             List<BigInteger> listOfDpns, Pair<IpAddress, String> pair)
 282             throws ExecutionException {
 283         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 284         if (newDesignatedDpn.equals(DHCPMConstants.INVALID_DPID)) {
 285             return;
 286         }
 287         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 288         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 289             logger.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 290             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs);
 291         }
 292     }
 293
 294     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId) {
 295         try {
 296             Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 297             if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 298                 return;
 299             }
 300             for (String vmMacAddress : setOfVmMacAddress) {
 301                 installDhcpDropAction(dpnId, vmMacAddress);
 302             }
 303         } catch (Exception e) {
 304             logger.error("Error in uninstallExistingFlows {}", e);
 305         }
 306     }
 307
 308     /**
 309      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 310      * @param tunnelIp
 311      * @param elanInstanceName
 312      * @param dpns
 313      * @return
 314      */
 315     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 316             List<BigInteger> dpns) {
 317         BigInteger designatedDpnId = DHCPMConstants.INVALID_DPID;
 318         if (dpns != null && dpns.size() != 0) {
 319             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 320             candidateDpns.retainAll(dpns);
 321             logger.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}", tunnelIp, elanInstanceName, candidateDpns);
 322             boolean elanDpnAvailableFlag = true;
 323             if (candidateDpns == null || candidateDpns.isEmpty()) {
 324                 candidateDpns = dpns;
 325                 elanDpnAvailableFlag = false;
 326             }
 327             int size = 0;
 328             L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 329             if (device == null) {
 330                 logger.trace("Could not find any device for elanInstanceName {} and tunnelIp {}", elanInstanceName, tunnelIp);
 331                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 332                 return designatedDpnId;
 333             }
 334             for (BigInteger dpn : candidateDpns) {
 335                 String hwvtepNodeId = device.getHwvtepNodeId();
 336                 if (!elanDpnAvailableFlag) {
 337                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 338                         logger.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 339                         continue;
 340                     }
 341                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 342                     logger.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 343                     continue;
 344                 }
 345                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 346                 if (tunnelIpElanNameSet == null) {
 347                     designatedDpnId = dpn;
 348                     break;
 349                 }
 350                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 351                     size = tunnelIpElanNameSet.size();
 352                     designatedDpnId = dpn;
 353                 }
 354             }
 355             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 356             return designatedDpnId;
 357         }
 358         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 359         return designatedDpnId;
 360     }
 361
 362     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 363         writeDesignatedSwitchForExternalTunnel(DHCPMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 364     }
 365
 366     public void installDhcpEntries(BigInteger dpnId, String vmMacAddress) {
 367         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil);
 368     }
 369
 370     public void installDhcpEntries(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 371         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 372         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 373                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 374                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 375         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 376             @Override
 377             public void onSuccess(Boolean isOwner) {
 378                 if (isOwner) {
 379                     installDhcpEntries(dpnId, vmMacAddress);
 380                 } else {
 381                     logger.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn {}", nodeId);
 382                 }
 383             }
 384
 385             @Override
 386             public void onFailure(Throwable error) {
 387                 logger.error("Error while fetching checkNodeEntityOwner", error);
 388             }
 389         });
 390     }
 391
 392     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress) {
 393         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil);
 394     }
 395
 396     public void unInstallDhcpEntries(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 397         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 398         // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 399         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 400                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 401                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 402         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 403             @Override
 404             public void onSuccess(Boolean isOwner) {
 405                 if (isOwner) {
 406                     unInstallDhcpEntries(dpnId, vmMacAddress);
 407                 } else {
 408                     logger.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn {}", nodeId);
 409                 }
 410             }
 411
 412             @Override
 413             public void onFailure(Throwable error) {
 414                 logger.error("Error while fetching checkNodeEntityOwner", error);
 415             }
 416         });
 417     }
 418
 419     public void installDhcpDropAction(BigInteger dpn, String vmMacAddress) {
 420         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil);
 421     }
 422
 423     public void installDhcpDropAction(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 424         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 425         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 426                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 427                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 428         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 429             @Override
 430             public void onSuccess(Boolean isOwner) {
 431                 if (isOwner) {
 432                     installDhcpDropAction(dpnId, vmMacAddress);
 433                 } else {
 434                     logger.trace("Exiting installDhcpDropAction since this cluster node is not the owner for dpn {}", nodeId);
 435                 }
 436             }
 437
 438             @Override
 439             public void onFailure(Throwable error) {
 440                 logger.error("Error while fetching checkNodeEntityOwner", error);
 441             }
 442         });
 443     }
 444
 445     public void handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn) {
 446         logger.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 447         if (interfaceDpn == null) {
 448             return;
 449         }
 450         try {
 451             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 452                 Set<Pair<IpAddress, String>> tunnelElanPairSet = designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 453                 if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 454                     return;
 455                 }
 456                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 457                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 458                     if (tunnelIpInDpn.equals(tunnelIp)) {
 459                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 460                             logger.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache", tunnelElanPair);
 461                             return;
 462                         }
 463                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 464                         dpns.remove(interfaceDpn);
 465                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn);
 466                         updateCacheAndInstallNewFlows(interfaceDpn, dpns, tunnelElanPair);
 467                     }
 468                 }
 469             }
 470         } catch (Exception e) {
 471             logger.error("Error in handleTunnelStateDown {}", e.getMessage());
 472             logger.trace("Exception details {}", e);
 473         }
 474     }
 475
 476     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 477         ConcurrentMap<String, L2GatewayDevice> l2GwDevices = ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 478         for (L2GatewayDevice device : l2GwDevices.values()) {
 479             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 480                 return true;
 481             }
 482         }
 483         return false;
 484     }
 485
 486     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 487         return new StringBuffer().append(tunnelIp.toString()).append(interfaceDpn).toString().intern();
 488     }
 489
 490     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 491         Set<Pair<IpAddress, String>> tunnelIpElanNameKeySet = tunnelIpElanNameToVmMacCache.keySet();
 492         for (Pair<IpAddress, String> pair : tunnelIpElanNameKeySet) {
 493             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 494                 Set<String> setOfExistingVmMacAddress;
 495                 setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 496                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 497                     continue;
 498                 }
 499                 logger.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress, setOfExistingVmMacAddress, elanInstanceName);
 500                 setOfExistingVmMacAddress.remove(vmMacAddress);
 501                 if (setOfExistingVmMacAddress.size() > 0) {
 502                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 503                     return;
 504                 }
 505                 tunnelIpElanNameToVmMacCache.remove(pair);
 506             }
 507         }
 508     }
 509
 510     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 511         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 512         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 513         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 514         if (tunnelIpElanNameSet != null) {
 515             logger.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for designatedDpnId {}", tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 516             tunnelIpElanNameSet.remove(tunnelIpElanName);
 517             if (tunnelIpElanNameSet.size() != 0) {
 518                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 519             } else {
 520                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 521             }
 522         }
 523     }
 524
 525     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 526         if (macAddress == null) {
 527             return;
 528         }
 529         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 530         logger.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni, macAddress.toUpperCase(), vniMacAddressPair, port);
 531         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 532     }
 533
 534     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 535         if (macAddress == null) {
 536             return;
 537         }
 538         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 539         vniMacAddressToPortCache.remove(vniMacAddressPair);
 540     }
 541
 542     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 543         if (macAddress == null) {
 544             return null;
 545         }
 546         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 547         logger.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni, macAddress.toUpperCase(), vniMacAddressPair, vniMacAddressToPortCache.get(vniMacAddressPair));
 548         return vniMacAddressToPortCache.get(vniMacAddressPair);
 549     }
 550
 551     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 552         String tunnelInterfaceName = null;
 553         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 554         try {
 555             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 556                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 557                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 558
 559             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 560             if (rpcResult.isSuccessful()) {
 561                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 562                 logger.debug("Tunnel interface name: {}", tunnelInterfaceName);
 563             } else {
 564                 logger.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}",
 565                         rpcResult.getErrors());
 566             }
 567         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 568             logger.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}: {} ",
 569                     sourceNode, dstNode, e);
 570         }
 571         return tunnelInterfaceName;
 572     }
 573
 574     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 575         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 576                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 577         Optional<Node> physicalSwitchOptional = MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 578         return physicalSwitchOptional;
 579     }
 580
 581     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 582         List<LocatorSet> locators = new ArrayList<>();
 583         for (TerminationPoint tp : dstDevice.getTerminationPoint()) {
 584             HwvtepPhysicalLocatorAugmentation aug = tp.getAugmentation(HwvtepPhysicalLocatorAugmentation.class);
 585             if (internalTunnelIp.getIpv4Address().equals(aug.getDstIp().getIpv4Address())) {
 586                 HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 587                         HwvtepSouthboundUtils.createPhysicalLocatorInstanceIdentifier(dstDevice.getNodeId(), aug));
 588                 locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 589             }
 590         }
 591         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 592                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 593
 594         RemoteMcastMacs remoteUcastMacs = new RemoteMcastMacsBuilder()
 595                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 596                 .setLogicalSwitchRef(lsRef).setLocatorSet(locators).build();
 597         return remoteUcastMacs;
 598     }
 599
 600     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName, L2GatewayDevice device, IpAddress internalTunnelIp) {
 601         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 602         Node dstNode = optionalNode.get();
 603         if (dstNode == null) {
 604             logger.debug("could not get device node {} ", device.getHwvtepNodeId());
 605             return null;
 606         }
 607         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 608         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 609         return transaction;
 610     }
 611
 612     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp, final String elanInstanceName) {
 613         if (designatedDpnId.equals(DHCPMConstants.INVALID_DPID)) {
 614             return;
 615         }
 616         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE, HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 617         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 618             @Override
 619             public void onSuccess(Boolean isOwner) {
 620                 if (isOwner) {
 621                     logger.info("Installing remote McastMac");
 622                     L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 623                     String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId), device.getHwvtepNodeId());
 624                     IpAddress internalTunnelIp = null;
 625                     if (tunnelInterfaceName != null) {
 626                         Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromConfigDS(tunnelInterfaceName, broker);
 627                         if (tunnelInterface == null) {
 628                             logger.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 629                             return;
 630                         }
 631                         internalTunnelIp = tunnelInterface.getAugmentation(IfTunnel.class).getTunnelSource();
 632                         WriteTransaction transaction = broker.newWriteOnlyTransaction();
 633                         putRemoteMcastMac(transaction, elanInstanceName, device, internalTunnelIp);
 634                         if (transaction != null) {
 635                             transaction.submit();
 636                         }
 637                     }
 638                 } else {
 639                       logger.info("Installing remote McastMac is not executed for this node.");
 640                 }
 641             }
 642
 643             @Override
 644             public void onFailure(Throwable error) {
 645                 logger.error("Failed to install remote McastMac", error);
 646             }
 647         });
 648     }
 649
 650     private L2GatewayDevice getDeviceFromTunnelIp(String elanInstanceName, IpAddress tunnelIp) {
 651         ConcurrentMap<String, L2GatewayDevice> devices = L2GatewayCacheUtils.getCache();
 652         logger.trace("In getDeviceFromTunnelIp devices {}", devices);
 653         for (L2GatewayDevice device : devices.values()) {
 654             if (tunnelIp.equals(device.getTunnelIp())) {
 655                 return device;
 656             }
 657         }
 658         return null;
 659     }
 660
 661     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 662         boolean isTunnelUp = false;
 663         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 664         if (tunnelInterfaceName == null) {
 665             logger.debug("Tunnel Interface is not present {}", tunnelInterfaceName);
 666             return isTunnelUp;
 667         }
 668         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 669         if (tunnelInterface == null) {
 670             logger.debug("Interface {} is not present in interface state", tunnelInterfaceName);
 671             return isTunnelUp;
 672         }
 673         isTunnelUp = (tunnelInterface.getOperStatus() == OperStatus.Up);
 674         return isTunnelUp;
 675     }
 676
 677     public void handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn) {
 678         logger.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 679         try {
 680             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 681                 Set<Pair<IpAddress, String>> tunnelIpElanPair = designatedDpnsToTunnelIpElanNameCache.get(DHCPMConstants.INVALID_DPID);
 682                 List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 683                 if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 684                     logger.trace("There are no undesignated DPNs");
 685                     return;
 686                 }
 687                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 688                     if (tunnelIp.equals(pair.getLeft())) {
 689                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, pair.getRight(), dpns);
 690                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DHCPMConstants.INVALID_DPID)) {
 691                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 692                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 693                                 logger.trace("Updating DHCP flow for macAddress {} with newDpn {}", vmMacAddress, newDesignatedDpn);
 694                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress);
 695                             }
 696                         }
 697                     }
 698                 }
 699             }
 700         } catch (Exception e) {
 701             logger.error("Error in handleTunnelStateUp {}", e.getMessage());
 702             logger.trace("Exception details {}", e);
 703         }
 704     }
 705
 706     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 707         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 708         if (tunnelInterfaceName == null) {
 709             return false;
 710         }
 711         Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromConfigDS(tunnelInterfaceName, broker);
 712         if (tunnelInterface == null) {
 713             logger.debug("Tunnel Interface is not present {}", tunnelInterfaceName);
 714             return false;
 715         }
 716         return true;
 717     }
 718
 719     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 720         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 721         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 722         logger.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 723                 elanInstanceName, tunnelIp, dpns, vmMacs);
 724         if (vmMacs == null) {
 725             return;
 726         }
 727         for (String vmMacAddress : vmMacs) {
 728             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 729         }
 730         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 731     }
 732
 733     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 734         availableVMCache.remove(tunnelIpElanName);
 735     }
 736
 737     private void unInstallDhcpEntriesOnDpns(List<BigInteger> dpns, String vmMacAddress) {
 738         for (BigInteger dpn : dpns) {
 739             unInstallDhcpEntries(dpn, vmMacAddress, entityOwnershipService);
 740         }
 741     }
 742
 743     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 744         if (logger.isTraceEnabled()) {
 745             logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName " + elanInstanceName);
 746         }
 747         IpAddress tunnelIp = null;
 748         Set<Pair<IpAddress, String>> tunnelElanKeySet = availableVMCache.keySet();
 749         Set<String> listExistingVmMacAddress;
 750         for (Pair<IpAddress, String> pair : tunnelElanKeySet) {
 751             if (logger.isTraceEnabled()) {
 752                 logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left  " + pair.getLeft() + " right:" +
 753                         pair.getRight());
 754             }
 755             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 756                 listExistingVmMacAddress = availableVMCache.get(pair);
 757                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty() &&
 758                         listExistingVmMacAddress.contains(vmMacAddress)) {
 759                     tunnelIp = pair.getLeft();
 760                     break;
 761                 }
 762             }
 763         }
 764         if (logger.isTraceEnabled()) {
 765             logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP " + tunnelIp);
 766         }
 767         return tunnelIp;
 768     }
 769 }