vpnservice/dhcpservice/dhcpservice-impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import java.math.BigInteger;
  11 import java.util.ArrayList;
  12 import java.util.Collection;
  13 import java.util.LinkedList;
  14 import java.util.List;
  15 import java.util.Set;
  16 import java.util.concurrent.ConcurrentHashMap;
  17 import java.util.concurrent.ConcurrentMap;
  18 import java.util.concurrent.CopyOnWriteArraySet;
  19 import java.util.concurrent.ExecutionException;
  20 import java.util.concurrent.Future;
  21
  22 import org.apache.commons.lang3.tuple.ImmutablePair;
  23 import org.apache.commons.lang3.tuple.Pair;
  24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  26 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  29 import org.opendaylight.genius.mdsalutil.NwConstants;
  30 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  31 import org.opendaylight.genius.utils.clustering.ClusteringUtils;
  32 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  33 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  34 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  35 import org.opendaylight.netvirt.dhcpservice.api.DHCPMConstants;
  36 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  37 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  38 import org.opendaylight.netvirt.neutronvpn.api.l2gw.utils.L2GatewayCacheUtils;
  39 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  41 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  42 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  43 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  44 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorAugmentation;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  67 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  68 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  69 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  70 import org.opendaylight.yangtools.yang.common.RpcResult;
  71 import org.slf4j.Logger;
  72 import org.slf4j.LoggerFactory;
  73
  74 import com.google.common.base.Optional;
  75 import com.google.common.util.concurrent.FutureCallback;
  76 import com.google.common.util.concurrent.Futures;
  77 import com.google.common.util.concurrent.ListenableFuture;
  78
  79 public class DhcpExternalTunnelManager {
  80
  81     private static final Logger logger = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
  82     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
  83     private final DataBroker broker;
  84     private IMdsalApiManager mdsalUtil;
  85
  86     private ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache = new ConcurrentHashMap<>();
  87     private ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache = new ConcurrentHashMap<>();
  88     private ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
  89     private ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
  90     private ItmRpcService itmRpcService;
  91     private EntityOwnershipService entityOwnershipService;
  92
  93     public DhcpExternalTunnelManager(DataBroker broker, IMdsalApiManager mdsalUtil, ItmRpcService itmRpcService, EntityOwnershipService entityOwnershipService) {
  94         this.broker = broker;
  95         this.mdsalUtil = mdsalUtil;
  96         this.itmRpcService = itmRpcService;
  97         this.entityOwnershipService = entityOwnershipService;
  98         initilizeCaches();
  99     }
 100
 101     private void initilizeCaches() {
 102         logger.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 103         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 104         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 105         if (designatedSwitchForTunnelOptional.isPresent()) {
 106             List<DesignatedSwitchForTunnel> list = designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 107             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 108                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair = designatedDpnsToTunnelIpElanNameCache.get(designatedSwitchForTunnel.getDpId());
 109                 if (setOfTunnelIpElanNamePair == null) {
 110                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 111                 }
 112                 Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(), designatedSwitchForTunnel.getElanInstanceName());
 113                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 114                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()), setOfTunnelIpElanNamePair);
 115             }
 116         }
 117         logger.trace("Loading vniMacAddressToPortCache");
 118         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 119         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 120         if (optionalPorts.isPresent()) {
 121             List<Port> list = optionalPorts.get().getPort();
 122             for (Port port : list) {
 123                 if(NeutronUtils.isPortVnicTypeNormal(port)) {
 124                     continue;
 125                 }
 126                 String macAddress = port.getMacAddress().getValue();
 127                 Uuid networkId = port.getNetworkId();
 128                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 129                 if (segmentationId == null) {
 130                     return;
 131                 }
 132                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 133             }
 134         }
 135
 136     }
 137
 138
 139     public BigInteger designateDpnId(IpAddress tunnelIp,
 140             String elanInstanceName, List<BigInteger> dpns) {
 141         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 142         if (designatedDpnId != null && !designatedDpnId.equals(DHCPMConstants.INVALID_DPID)) {
 143             logger.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp, elanInstanceName);
 144             return designatedDpnId;
 145         }
 146         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 147     }
 148
 149     public void installDhcpFlowsForVms(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns,
 150             BigInteger designatedDpnId, String vmMacAddress ) {
 151         logger.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp, elanInstanceName, designatedDpnId, vmMacAddress);
 152         synchronized (getTunnelIpDpnKey(tunnelIp, designatedDpnId)) {
 153             installDhcpEntries(designatedDpnId, vmMacAddress, entityOwnershipService);
 154             dpns.remove(designatedDpnId);
 155             for (BigInteger dpn : dpns) {
 156                 installDhcpDropAction(dpn, vmMacAddress, entityOwnershipService);
 157             }
 158             updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 159         }
 160     }
 161
 162     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress) {
 163         for (String vmMacAddress : listVmMacAddress) {
 164             installDhcpEntries(designatedDpnId, vmMacAddress);
 165         }
 166     }
 167
 168     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 169         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 170         removeFromLocalCache(elanInstanceName, vmMacAddress);
 171     }
 172
 173     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 174         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 175             return null;
 176         }
 177         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 178         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 179         if (designatedSwitchForTunnelOptional.isPresent()) {
 180             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 181         }
 182         return null;
 183     }
 184
 185     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 186         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 187         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 188         DesignatedSwitchForTunnel designatedSwitchForTunnel = new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue()).setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp).setKey(designatedSwitchForTunnelKey).build();
 189         logger.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName, dpnId);
 190         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 191         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 192     }
 193
 194     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 195         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 196         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 197         logger.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName, dpnId);
 198         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 199         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 200     }
 201
 202     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 203         List<String> vmMacs = getAllVmMacs();
 204         logger.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 205         for (String vmMacAddress : vmMacs) {
 206             installDhcpDropAction(dpId, vmMacAddress);
 207         }
 208     }
 209
 210     private List<String> getAllVmMacs() {
 211         List<String> vmMacs = new LinkedList<>();
 212         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 213         for (Set<String> list : listOfVmMacs) {
 214             vmMacs.addAll(list);
 215         }
 216         return vmMacs;
 217     }
 218
 219     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 220         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 221         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 222         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 223         if (tunnelIpElanNameSet == null) {
 224             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 225         }
 226         tunnelIpElanNameSet.add(tunnelIpElanName);
 227         logger.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId, tunnelIpElanNameSet);
 228         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 229     }
 230
 231     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 232         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 233         Set<String> setOfExistingVmMacAddress;
 234         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 235         if (setOfExistingVmMacAddress == null) {
 236             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 237         }
 238         setOfExistingVmMacAddress.add(vmMacAddress);
 239         logger.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName, setOfExistingVmMacAddress);
 240         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 241         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 242     }
 243
 244     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 245         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<IpAddress, String>(tunnelIp, elanInstanceName);
 246         Set<String> listExistingVmMacAddress;
 247         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 248         if (null == listExistingVmMacAddress) {
 249             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 250         }
 251         listExistingVmMacAddress.add(vmMacAddress);
 252         logger.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 253                 listExistingVmMacAddress);
 254         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 255     }
 256
 257     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 258         logger.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 259         try {
 260             Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 261             if (!dpnId.equals(DHCPMConstants.INVALID_DPID)) {
 262                 List<String> listOfVms = getAllVmMacs();
 263                 for (String vmMacAddress : listOfVms) {
 264                     unInstallDhcpEntries(dpnId, vmMacAddress);
 265                 }
 266             }
 267             if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 268                 logger.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 269                 return;
 270             }
 271             for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 272                 updateCacheAndInstallNewFlows(dpnId, listOfDpns, pair);
 273             }
 274         } catch (Exception e) {
 275             logger.error("Error in handleDesignatedDpnDown {}", e);
 276         }
 277     }
 278
 279     public void updateCacheAndInstallNewFlows(BigInteger dpnId,
 280             List<BigInteger> listOfDpns, Pair<IpAddress, String> pair)
 281             throws ExecutionException {
 282         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 283         if (newDesignatedDpn.equals(DHCPMConstants.INVALID_DPID)) {
 284             return;
 285         }
 286         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 287         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 288             logger.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 289             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs);
 290         }
 291     }
 292
 293     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId) {
 294         try {
 295             Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 296             if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 297                 return;
 298             }
 299             for (String vmMacAddress : setOfVmMacAddress) {
 300                 installDhcpDropAction(dpnId, vmMacAddress);
 301             }
 302         } catch (Exception e) {
 303             logger.error("Error in uninstallExistingFlows {}", e);
 304         }
 305     }
 306
 307     /**
 308      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 309      * @param tunnelIp
 310      * @param elanInstanceName
 311      * @param dpns
 312      * @return
 313      */
 314     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 315             List<BigInteger> dpns) {
 316         BigInteger designatedDpnId = DHCPMConstants.INVALID_DPID;
 317         if (dpns != null && dpns.size() != 0) {
 318             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 319             candidateDpns.retainAll(dpns);
 320             logger.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}", tunnelIp, elanInstanceName, candidateDpns);
 321             boolean elanDpnAvailableFlag = true;
 322             if (candidateDpns == null || candidateDpns.isEmpty()) {
 323                 candidateDpns = dpns;
 324                 elanDpnAvailableFlag = false;
 325             }
 326             int size = 0;
 327             L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 328             if (device == null) {
 329                 logger.trace("Could not find any device for elanInstanceName {} and tunnelIp {}", elanInstanceName, tunnelIp);
 330                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 331                 return designatedDpnId;
 332             }
 333             for (BigInteger dpn : candidateDpns) {
 334                 String hwvtepNodeId = device.getHwvtepNodeId();
 335                 if (!elanDpnAvailableFlag) {
 336                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 337                         logger.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 338                         continue;
 339                     }
 340                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 341                     logger.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 342                     continue;
 343                 }
 344                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 345                 if (tunnelIpElanNameSet == null) {
 346                     designatedDpnId = dpn;
 347                     break;
 348                 }
 349                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 350                     size = tunnelIpElanNameSet.size();
 351                     designatedDpnId = dpn;
 352                 }
 353             }
 354             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 355             return designatedDpnId;
 356         }
 357         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 358         return designatedDpnId;
 359     }
 360
 361     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 362         writeDesignatedSwitchForExternalTunnel(DHCPMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 363     }
 364
 365     public void installDhcpEntries(BigInteger dpnId, String vmMacAddress) {
 366         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil);
 367     }
 368
 369     public void installDhcpEntries(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 370         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 371         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 372                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 373                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 374         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 375             @Override
 376             public void onSuccess(Boolean isOwner) {
 377                 if (isOwner) {
 378                     installDhcpEntries(dpnId, vmMacAddress);
 379                 } else {
 380                     logger.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn {}", nodeId);
 381                 }
 382             }
 383
 384             @Override
 385             public void onFailure(Throwable error) {
 386                 logger.error("Error while fetching checkNodeEntityOwner", error);
 387             }
 388         });
 389     }
 390
 391     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress) {
 392         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil);
 393     }
 394
 395     public void unInstallDhcpEntries(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 396         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 397         // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 398         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 399                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 400                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 401         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 402             @Override
 403             public void onSuccess(Boolean isOwner) {
 404                 if (isOwner) {
 405                     unInstallDhcpEntries(dpnId, vmMacAddress);
 406                 } else {
 407                     logger.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn {}", nodeId);
 408                 }
 409             }
 410
 411             @Override
 412             public void onFailure(Throwable error) {
 413                 logger.error("Error while fetching checkNodeEntityOwner", error);
 414             }
 415         });
 416     }
 417
 418     public void installDhcpDropAction(BigInteger dpn, String vmMacAddress) {
 419         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil);
 420     }
 421
 422     public void installDhcpDropAction(final BigInteger dpnId, final String vmMacAddress, EntityOwnershipService eos) {
 423         final String nodeId = DhcpServiceUtils.getNodeIdFromDpnId(dpnId);
 424         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 425                 eos, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 426                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 427         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 428             @Override
 429             public void onSuccess(Boolean isOwner) {
 430                 if (isOwner) {
 431                     installDhcpDropAction(dpnId, vmMacAddress);
 432                 } else {
 433                     logger.trace("Exiting installDhcpDropAction since this cluster node is not the owner for dpn {}", nodeId);
 434                 }
 435             }
 436
 437             @Override
 438             public void onFailure(Throwable error) {
 439                 logger.error("Error while fetching checkNodeEntityOwner", error);
 440             }
 441         });
 442     }
 443
 444     public void handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn) {
 445         logger.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 446         if (interfaceDpn == null) {
 447             return;
 448         }
 449         try {
 450             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 451                 Set<Pair<IpAddress, String>> tunnelElanPairSet = designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 452                 if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 453                     return;
 454                 }
 455                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 456                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 457                     if (tunnelIpInDpn.equals(tunnelIp)) {
 458                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 459                             logger.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache", tunnelElanPair);
 460                             return;
 461                         }
 462                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 463                         dpns.remove(interfaceDpn);
 464                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn);
 465                         updateCacheAndInstallNewFlows(interfaceDpn, dpns, tunnelElanPair);
 466                     }
 467                 }
 468             }
 469         } catch (Exception e) {
 470             logger.error("Error in handleTunnelStateDown {}", e.getMessage());
 471             logger.trace("Exception details {}", e);
 472         }
 473     }
 474
 475     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 476         ConcurrentMap<String, L2GatewayDevice> l2GwDevices = ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 477         for (L2GatewayDevice device : l2GwDevices.values()) {
 478             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 479                 return true;
 480             }
 481         }
 482         return false;
 483     }
 484
 485     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 486         return new StringBuffer().append(tunnelIp.toString()).append(interfaceDpn).toString().intern();
 487     }
 488
 489     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 490         Set<Pair<IpAddress, String>> tunnelIpElanNameKeySet = tunnelIpElanNameToVmMacCache.keySet();
 491         for (Pair<IpAddress, String> pair : tunnelIpElanNameKeySet) {
 492             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 493                 Set<String> setOfExistingVmMacAddress;
 494                 setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 495                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 496                     continue;
 497                 }
 498                 logger.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress, setOfExistingVmMacAddress, elanInstanceName);
 499                 setOfExistingVmMacAddress.remove(vmMacAddress);
 500                 if (setOfExistingVmMacAddress.size() > 0) {
 501                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 502                     return;
 503                 }
 504                 tunnelIpElanNameToVmMacCache.remove(pair);
 505             }
 506         }
 507     }
 508
 509     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 510         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 511         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 512         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 513         if (tunnelIpElanNameSet != null) {
 514             logger.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for designatedDpnId {}", tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 515             tunnelIpElanNameSet.remove(tunnelIpElanName);
 516             if (tunnelIpElanNameSet.size() != 0) {
 517                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 518             } else {
 519                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 520             }
 521         }
 522     }
 523
 524     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 525         if (macAddress == null) {
 526             return;
 527         }
 528         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 529         logger.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni, macAddress.toUpperCase(), vniMacAddressPair, port);
 530         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 531     }
 532
 533     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 534         if (macAddress == null) {
 535             return;
 536         }
 537         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 538         vniMacAddressToPortCache.remove(vniMacAddressPair);
 539     }
 540
 541     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 542         if (macAddress == null) {
 543             return null;
 544         }
 545         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 546         logger.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni, macAddress.toUpperCase(), vniMacAddressPair, vniMacAddressToPortCache.get(vniMacAddressPair));
 547         return vniMacAddressToPortCache.get(vniMacAddressPair);
 548     }
 549
 550     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 551         String tunnelInterfaceName = null;
 552         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 553         try {
 554             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 555                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 556                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 557
 558             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 559             if (rpcResult.isSuccessful()) {
 560                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 561                 logger.debug("Tunnel interface name: {}", tunnelInterfaceName);
 562             } else {
 563                 logger.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}",
 564                         rpcResult.getErrors());
 565             }
 566         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 567             logger.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}: {} ",
 568                     sourceNode, dstNode, e);
 569         }
 570         return tunnelInterfaceName;
 571     }
 572
 573     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 574         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 575                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 576         Optional<Node> physicalSwitchOptional = MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 577         return physicalSwitchOptional;
 578     }
 579
 580     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 581         List<LocatorSet> locators = new ArrayList<>();
 582         for (TerminationPoint tp : dstDevice.getTerminationPoint()) {
 583             HwvtepPhysicalLocatorAugmentation aug = tp.getAugmentation(HwvtepPhysicalLocatorAugmentation.class);
 584             if (internalTunnelIp.getIpv4Address().equals(aug.getDstIp().getIpv4Address())) {
 585                 HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 586                         HwvtepSouthboundUtils.createPhysicalLocatorInstanceIdentifier(dstDevice.getNodeId(), aug));
 587                 locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 588             }
 589         }
 590         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 591                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 592
 593         RemoteMcastMacs remoteUcastMacs = new RemoteMcastMacsBuilder()
 594                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 595                 .setLogicalSwitchRef(lsRef).setLocatorSet(locators).build();
 596         return remoteUcastMacs;
 597     }
 598
 599     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName, L2GatewayDevice device, IpAddress internalTunnelIp) {
 600         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 601         Node dstNode = optionalNode.get();
 602         if (dstNode == null) {
 603             logger.debug("could not get device node {} ", device.getHwvtepNodeId());
 604             return null;
 605         }
 606         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 607         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 608         return transaction;
 609     }
 610
 611     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp, final String elanInstanceName) {
 612         if (designatedDpnId.equals(DHCPMConstants.INVALID_DPID)) {
 613             return;
 614         }
 615         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE, HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 616         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 617             @Override
 618             public void onSuccess(Boolean isOwner) {
 619                 if (isOwner) {
 620                     logger.info("Installing remote McastMac");
 621                     L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 622                     String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId), device.getHwvtepNodeId());
 623                     IpAddress internalTunnelIp = null;
 624                     if (tunnelInterfaceName != null) {
 625                         Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromConfigDS(tunnelInterfaceName, broker);
 626                         if (tunnelInterface == null) {
 627                             logger.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 628                             return;
 629                         }
 630                         internalTunnelIp = tunnelInterface.getAugmentation(IfTunnel.class).getTunnelSource();
 631                         WriteTransaction transaction = broker.newWriteOnlyTransaction();
 632                         putRemoteMcastMac(transaction, elanInstanceName, device, internalTunnelIp);
 633                         if (transaction != null) {
 634                             transaction.submit();
 635                         }
 636                     }
 637                 } else {
 638                       logger.info("Installing remote McastMac is not executed for this node.");
 639                 }
 640             }
 641
 642             @Override
 643             public void onFailure(Throwable error) {
 644                 logger.error("Failed to install remote McastMac", error);
 645             }
 646         });
 647     }
 648
 649     private L2GatewayDevice getDeviceFromTunnelIp(String elanInstanceName, IpAddress tunnelIp) {
 650         ConcurrentMap<String, L2GatewayDevice> devices = L2GatewayCacheUtils.getCache();
 651         logger.trace("In getDeviceFromTunnelIp devices {}", devices);
 652         for (L2GatewayDevice device : devices.values()) {
 653             if (tunnelIp.equals(device.getTunnelIp())) {
 654                 return device;
 655             }
 656         }
 657         return null;
 658     }
 659
 660     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 661         boolean isTunnelUp = false;
 662         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 663         if (tunnelInterfaceName == null) {
 664             logger.debug("Tunnel Interface is not present {}", tunnelInterfaceName);
 665             return isTunnelUp;
 666         }
 667         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 668         if (tunnelInterface == null) {
 669             logger.debug("Interface {} is not present in interface state", tunnelInterfaceName);
 670             return isTunnelUp;
 671         }
 672         isTunnelUp = (tunnelInterface.getOperStatus() == OperStatus.Up);
 673         return isTunnelUp;
 674     }
 675
 676     public void handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn) {
 677         logger.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 678         try {
 679             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 680                 Set<Pair<IpAddress, String>> tunnelIpElanPair = designatedDpnsToTunnelIpElanNameCache.get(DHCPMConstants.INVALID_DPID);
 681                 List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 682                 if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 683                     logger.trace("There are no undesignated DPNs");
 684                     return;
 685                 }
 686                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 687                     if (tunnelIp.equals(pair.getLeft())) {
 688                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, pair.getRight(), dpns);
 689                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DHCPMConstants.INVALID_DPID)) {
 690                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 691                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 692                                 logger.trace("Updating DHCP flow for macAddress {} with newDpn {}", vmMacAddress, newDesignatedDpn);
 693                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress);
 694                             }
 695                         }
 696                     }
 697                 }
 698             }
 699         } catch (Exception e) {
 700             logger.error("Error in handleTunnelStateUp {}", e.getMessage());
 701             logger.trace("Exception details {}", e);
 702         }
 703     }
 704
 705     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 706         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 707         if (tunnelInterfaceName == null) {
 708             return false;
 709         }
 710         Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromConfigDS(tunnelInterfaceName, broker);
 711         if (tunnelInterface == null) {
 712             logger.debug("Tunnel Interface is not present {}", tunnelInterfaceName);
 713             return false;
 714         }
 715         return true;
 716     }
 717
 718     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 719         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 720         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 721         logger.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 722                 elanInstanceName, tunnelIp, dpns, vmMacs);
 723         if (vmMacs == null) {
 724             return;
 725         }
 726         for (String vmMacAddress : vmMacs) {
 727             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 728         }
 729         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 730     }
 731
 732     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 733         availableVMCache.remove(tunnelIpElanName);
 734     }
 735
 736     private void unInstallDhcpEntriesOnDpns(List<BigInteger> dpns, String vmMacAddress) {
 737         for (BigInteger dpn : dpns) {
 738             unInstallDhcpEntries(dpn, vmMacAddress, entityOwnershipService);
 739         }
 740     }
 741
 742     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 743         if (logger.isTraceEnabled()) {
 744             logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName " + elanInstanceName);
 745         }
 746         IpAddress tunnelIp = null;
 747         Set<Pair<IpAddress, String>> tunnelElanKeySet = availableVMCache.keySet();
 748         Set<String> listExistingVmMacAddress;
 749         for (Pair<IpAddress, String> pair : tunnelElanKeySet) {
 750             if (logger.isTraceEnabled()) {
 751                 logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left  " + pair.getLeft() + " right:" +
 752                         pair.getRight());
 753             }
 754             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 755                 listExistingVmMacAddress = availableVMCache.get(pair);
 756                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty() &&
 757                         listExistingVmMacAddress.contains(vmMacAddress)) {
 758                     tunnelIp = pair.getLeft();
 759                     break;
 760                 }
 761             }
 762         }
 763         if (logger.isTraceEnabled()) {
 764             logger.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP " + tunnelIp);
 765         }
 766         return tunnelIp;
 767     }
 768 }