vpnservice/dhcpservice/dhcpservice-impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.FutureCallback;
  12 import com.google.common.util.concurrent.Futures;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14
  15 import java.math.BigInteger;
  16 import java.util.ArrayList;
  17 import java.util.Collection;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Set;
  21 import java.util.concurrent.ConcurrentHashMap;
  22 import java.util.concurrent.ConcurrentMap;
  23 import java.util.concurrent.CopyOnWriteArraySet;
  24 import java.util.concurrent.ExecutionException;
  25 import java.util.concurrent.Future;
  26
  27 import org.apache.commons.lang3.tuple.ImmutablePair;
  28 import org.apache.commons.lang3.tuple.Pair;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  31 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  34 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  35 import org.opendaylight.genius.mdsalutil.NwConstants;
  36 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  37 import org.opendaylight.genius.utils.clustering.ClusteringUtils;
  38 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  39 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  40 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  41 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  42 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  43 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  44 import org.opendaylight.netvirt.neutronvpn.api.l2gw.utils.L2GatewayCacheUtils;
  45 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  46 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  47 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  50 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorAugmentation;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  73 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  74 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  75 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  76 import org.opendaylight.yangtools.yang.common.RpcResult;
  77 import org.slf4j.Logger;
  78 import org.slf4j.LoggerFactory;
  79
  80 public class DhcpExternalTunnelManager {
  81
  82     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
  83     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
  84
  85     private final DataBroker broker;
  86     private final IMdsalApiManager mdsalUtil;
  87     private final ItmRpcService itmRpcService;
  88     private final EntityOwnershipService entityOwnershipService;
  89     private final IInterfaceManager interfaceManager;
  90
  91     private final ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
  92             new ConcurrentHashMap<>();
  93     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
  94             new ConcurrentHashMap<>();
  95     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
  96     private final ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
  97
  98
  99     public DhcpExternalTunnelManager(final DataBroker broker,
 100             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 101             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager) {
 102         this.broker = broker;
 103         this.mdsalUtil = mdsalUtil;
 104         this.itmRpcService = itmRpcService;
 105         this.entityOwnershipService = entityOwnershipService;
 106         this.interfaceManager = interfaceManager;
 107     }
 108
 109     public void init() {
 110         initilizeCaches();
 111     }
 112
 113     private void initilizeCaches() {
 114         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 115         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 116                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 117         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 118                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 119         if (designatedSwitchForTunnelOptional.isPresent()) {
 120             List<DesignatedSwitchForTunnel> list =
 121                     designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 122             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 123                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 124                         designatedDpnsToTunnelIpElanNameCache.get(designatedSwitchForTunnel.getDpId());
 125                 if (setOfTunnelIpElanNamePair == null) {
 126                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 127                 }
 128                 Pair<IpAddress, String> tunnelIpElanNamePair =
 129                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 130                                 designatedSwitchForTunnel.getElanInstanceName());
 131                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 132                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()),
 133                         setOfTunnelIpElanNamePair);
 134             }
 135         }
 136         LOG.trace("Loading vniMacAddressToPortCache");
 137         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 138         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 139         if (optionalPorts.isPresent()) {
 140             List<Port> list = optionalPorts.get().getPort();
 141             for (Port port : list) {
 142                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 143                     continue;
 144                 }
 145                 String macAddress = port.getMacAddress().getValue();
 146                 Uuid networkId = port.getNetworkId();
 147                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 148                 if (segmentationId == null) {
 149                     return;
 150                 }
 151                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 152             }
 153         }
 154     }
 155
 156     public BigInteger designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns) {
 157         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 158         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 159             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 160                     elanInstanceName);
 161             return designatedDpnId;
 162         }
 163         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 164     }
 165
 166     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<BigInteger> dpns,
 167             final BigInteger designatedDpnId, final String vmMacAddress) {
 168         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 169                 elanInstanceName, designatedDpnId, vmMacAddress);
 170         // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 171         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 172                 entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 173                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 174         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 175             @Override
 176             public void onSuccess(Boolean isOwner) {
 177                 if (isOwner) {
 178                     synchronized (getTunnelIpDpnKey(tunnelIp, designatedDpnId)) {
 179                         WriteTransaction tx = broker.newWriteOnlyTransaction();
 180                         dpns.remove(designatedDpnId);
 181                         for (BigInteger dpn : dpns) {
 182                             installDhcpDropAction(dpn, vmMacAddress, tx);
 183                         }
 184                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 185                         DhcpServiceUtils.submitTransaction(tx);
 186                     }
 187                 } else {
 188                     LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 189                 }
 190             }
 191
 192             @Override
 193             public void onFailure(Throwable error) {
 194                 LOG.error("Error while fetching checkNodeEntityOwner", error);
 195             }
 196         });
 197         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 198     }
 199
 200     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress, WriteTransaction tx) {
 201         for (String vmMacAddress : listVmMacAddress) {
 202             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 203         }
 204     }
 205
 206     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 207         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 208         removeFromLocalCache(elanInstanceName, vmMacAddress);
 209     }
 210
 211     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 212         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 213         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 214         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 215                 elanInstanceName, tunnelIp, dpns, vmMacs);
 216         if (vmMacs == null) {
 217             return;
 218         }
 219         for (String vmMacAddress : vmMacs) {
 220             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 221         }
 222         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 223     }
 224
 225     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 226         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 227             return null;
 228         }
 229         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 230                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 231                         .child(DesignatedSwitchForTunnel.class,
 232                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 233         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional =
 234                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 235         if (designatedSwitchForTunnelOptional.isPresent()) {
 236             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 237         }
 238         return null;
 239     }
 240
 241     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 242                                                        String elanInstanceName) {
 243         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 244                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 245         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 246                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 247                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 248         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 249                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 250                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 251                         .setKey(designatedSwitchForTunnelKey).build();
 252         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 253                 dpnId);
 254         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 255         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 256     }
 257
 258     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 259                                                         String elanInstanceName) {
 260         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 261                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 262         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 263                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 264                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 265         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 266                 elanInstanceName, dpnId);
 267         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 268         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 269     }
 270
 271     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 272         List<String> vmMacs = getAllVmMacs();
 273         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 274         WriteTransaction tx = broker.newWriteOnlyTransaction();
 275         for (String vmMacAddress : vmMacs) {
 276             installDhcpDropAction(dpId, vmMacAddress, tx);
 277         }
 278         DhcpServiceUtils.submitTransaction(tx);
 279     }
 280
 281     private List<String> getAllVmMacs() {
 282         List<String> vmMacs = new LinkedList<>();
 283         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 284         for (Set<String> list : listOfVmMacs) {
 285             vmMacs.addAll(list);
 286         }
 287         return vmMacs;
 288     }
 289
 290     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 291         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 292         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 293         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 294         if (tunnelIpElanNameSet == null) {
 295             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 296         }
 297         tunnelIpElanNameSet.add(tunnelIpElanName);
 298         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 299                 tunnelIpElanNameSet);
 300         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 301     }
 302
 303     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 304         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 305         Set<String> setOfExistingVmMacAddress;
 306         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 307         if (setOfExistingVmMacAddress == null) {
 308             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 309         }
 310         setOfExistingVmMacAddress.add(vmMacAddress);
 311         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 312                 setOfExistingVmMacAddress);
 313         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 314         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 315     }
 316
 317     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 318         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<IpAddress, String>(tunnelIp, elanInstanceName);
 319         Set<String> listExistingVmMacAddress;
 320         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 321         if (listExistingVmMacAddress == null) {
 322             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 323         }
 324         listExistingVmMacAddress.add(vmMacAddress);
 325         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 326                 listExistingVmMacAddress);
 327         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 328     }
 329
 330     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 331         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 332         try {
 333             Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 334             WriteTransaction tx = broker.newWriteOnlyTransaction();
 335             if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 336                 List<String> listOfVms = getAllVmMacs();
 337                 for (String vmMacAddress : listOfVms) {
 338                     unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 339                 }
 340             }
 341             if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 342                 LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 343                 return;
 344             }
 345             for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 346                 updateCacheAndInstallNewFlows(dpnId, listOfDpns, pair, tx);
 347             }
 348             DhcpServiceUtils.submitTransaction(tx);
 349         } catch (ExecutionException e) {
 350             LOG.error("Error in handleDesignatedDpnDown {}", e);
 351         }
 352     }
 353
 354     public void updateCacheAndInstallNewFlows(BigInteger dpnId,
 355             List<BigInteger> listOfDpns, Pair<IpAddress, String> pair, WriteTransaction tx)
 356             throws ExecutionException {
 357         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 358         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 359             return;
 360         }
 361         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 362         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 363             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 364             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 365         }
 366     }
 367
 368     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId,
 369                                           WriteTransaction tx) {
 370         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 371         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 372             return;
 373         }
 374         for (String vmMacAddress : setOfVmMacAddress) {
 375             installDhcpDropAction(dpnId, vmMacAddress, tx);
 376         }
 377     }
 378
 379     /**
 380      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 381      * @param tunnelIp The tunnel Ip address
 382      * @param elanInstanceName The elan instance name
 383      * @param dpns The data path nodes
 384      * @return The designated dpn
 385      */
 386     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 387             List<BigInteger> dpns) {
 388         BigInteger designatedDpnId = DhcpMConstants.INVALID_DPID;
 389         if (dpns != null && dpns.size() != 0) {
 390             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 391             candidateDpns.retainAll(dpns);
 392             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 393                     tunnelIp, elanInstanceName, candidateDpns);
 394             boolean elanDpnAvailableFlag = true;
 395             if (candidateDpns == null || candidateDpns.isEmpty()) {
 396                 candidateDpns = dpns;
 397                 elanDpnAvailableFlag = false;
 398             }
 399             int size = 0;
 400             L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 401             if (device == null) {
 402                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 403                         elanInstanceName, tunnelIp);
 404                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 405                 return designatedDpnId;
 406             }
 407             for (BigInteger dpn : candidateDpns) {
 408                 String hwvtepNodeId = device.getHwvtepNodeId();
 409                 if (!elanDpnAvailableFlag) {
 410                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 411                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 412                         continue;
 413                     }
 414                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 415                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 416                     continue;
 417                 }
 418                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 419                 if (tunnelIpElanNameSet == null) {
 420                     designatedDpnId = dpn;
 421                     break;
 422                 }
 423                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 424                     size = tunnelIpElanNameSet.size();
 425                     designatedDpnId = dpn;
 426                 }
 427             }
 428             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 429             return designatedDpnId;
 430         }
 431         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 432         return designatedDpnId;
 433     }
 434
 435     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 436         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 437     }
 438
 439     private void installDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 440         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 441                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 442     }
 443
 444     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 445         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 446                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, tx);
 447     }
 448
 449     private void installDhcpDropAction(BigInteger dpn, String vmMacAddress, WriteTransaction tx) {
 450         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 451                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 452     }
 453
 454     public void handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn,
 455                                       List<ListenableFuture<Void>> futures) {
 456         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 457         if (interfaceDpn == null) {
 458             return;
 459         }
 460         try {
 461             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 462                 Set<Pair<IpAddress, String>> tunnelElanPairSet =
 463                         designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 464                 if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 465                     return;
 466                 }
 467                 WriteTransaction tx = broker.newWriteOnlyTransaction();
 468                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 469                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 470                     if (tunnelIpInDpn.equals(tunnelIp)) {
 471                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 472                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 473                                     tunnelElanPair);
 474                             return;
 475                         }
 476                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 477                         dpns.remove(interfaceDpn);
 478                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 479                         updateCacheAndInstallNewFlows(interfaceDpn, dpns, tunnelElanPair, tx);
 480                     }
 481                 }
 482                 futures.add(tx.submit());
 483             }
 484         } catch (ExecutionException e) {
 485             LOG.error("Error in handleTunnelStateDown {}", e.getMessage());
 486             LOG.trace("Exception details {}", e);
 487         }
 488     }
 489
 490     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 491         ConcurrentMap<String, L2GatewayDevice> l2GwDevices =
 492                 ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 493         for (L2GatewayDevice device : l2GwDevices.values()) {
 494             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 495                 return true;
 496             }
 497         }
 498         return false;
 499     }
 500
 501     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 502         return tunnelIp.toString() + interfaceDpn;
 503     }
 504
 505     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 506         Set<Pair<IpAddress, String>> tunnelIpElanNameKeySet = tunnelIpElanNameToVmMacCache.keySet();
 507         for (Pair<IpAddress, String> pair : tunnelIpElanNameKeySet) {
 508             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 509                 Set<String> setOfExistingVmMacAddress;
 510                 setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 511                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 512                     continue;
 513                 }
 514                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 515                         setOfExistingVmMacAddress, elanInstanceName);
 516                 setOfExistingVmMacAddress.remove(vmMacAddress);
 517                 if (setOfExistingVmMacAddress.size() > 0) {
 518                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 519                     return;
 520                 }
 521                 tunnelIpElanNameToVmMacCache.remove(pair);
 522             }
 523         }
 524     }
 525
 526     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 527         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 528         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 529         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 530         if (tunnelIpElanNameSet != null) {
 531             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 532                             + "designatedDpnId {}",
 533                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 534             tunnelIpElanNameSet.remove(tunnelIpElanName);
 535             if (tunnelIpElanNameSet.size() != 0) {
 536                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 537             } else {
 538                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 539             }
 540         }
 541     }
 542
 543     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 544         if (macAddress == null) {
 545             return;
 546         }
 547         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 548         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 549                 macAddress.toUpperCase(), vniMacAddressPair, port);
 550         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 551     }
 552
 553     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 554         if (macAddress == null) {
 555             return;
 556         }
 557         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 558         vniMacAddressToPortCache.remove(vniMacAddressPair);
 559     }
 560
 561     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 562         if (macAddress == null) {
 563             return null;
 564         }
 565         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 566         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 567                 vni, macAddress.toUpperCase(), vniMacAddressPair, vniMacAddressToPortCache.get(vniMacAddressPair));
 568         return vniMacAddressToPortCache.get(vniMacAddressPair);
 569     }
 570
 571     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 572         String tunnelInterfaceName = null;
 573         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 574         try {
 575             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 576                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 577                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 578
 579             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 580             if (rpcResult.isSuccessful()) {
 581                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 582                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 583             } else {
 584                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 585             }
 586         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 587             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}: {} ",
 588                     sourceNode, dstNode, e);
 589         }
 590         return tunnelInterfaceName;
 591     }
 592
 593     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 594         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 595                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 596         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 597     }
 598
 599     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 600         List<LocatorSet> locators = new ArrayList<>();
 601         for (TerminationPoint tp : dstDevice.getTerminationPoint()) {
 602             HwvtepPhysicalLocatorAugmentation aug = tp.getAugmentation(HwvtepPhysicalLocatorAugmentation.class);
 603             if (internalTunnelIp.getIpv4Address().equals(aug.getDstIp().getIpv4Address())) {
 604                 HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 605                         HwvtepSouthboundUtils.createPhysicalLocatorInstanceIdentifier(dstDevice.getNodeId(), aug));
 606                 locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 607             }
 608         }
 609         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 610                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 611
 612         RemoteMcastMacs remoteUcastMacs = new RemoteMcastMacsBuilder()
 613                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 614                 .setLogicalSwitchRef(lsRef).setLocatorSet(locators).build();
 615         return remoteUcastMacs;
 616     }
 617
 618     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName,
 619                                                L2GatewayDevice device, IpAddress internalTunnelIp) {
 620         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 621         Node dstNode = optionalNode.get();
 622         if (dstNode == null) {
 623             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 624             return null;
 625         }
 626         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 627         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 628         return transaction;
 629     }
 630
 631     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp,
 632                                       final String elanInstanceName) {
 633         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 634             return;
 635         }
 636         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(entityOwnershipService,
 637                 HwvtepSouthboundConstants.ELAN_ENTITY_TYPE, HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 638         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 639             @Override
 640             public void onSuccess(Boolean isOwner) {
 641                 if (isOwner) {
 642                     LOG.info("Installing remote McastMac");
 643                     L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 644                     String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 645                             device.getHwvtepNodeId());
 646                     IpAddress internalTunnelIp = null;
 647                     if (tunnelInterfaceName != null) {
 648                         Interface tunnelInterface =
 649                                 interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 650                         if (tunnelInterface == null) {
 651                             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 652                             return;
 653                         }
 654                         internalTunnelIp = tunnelInterface.getAugmentation(IfTunnel.class).getTunnelSource();
 655                         WriteTransaction transaction = broker.newWriteOnlyTransaction();
 656                         putRemoteMcastMac(transaction, elanInstanceName, device, internalTunnelIp);
 657                         if (transaction != null) {
 658                             transaction.submit();
 659                         }
 660                     }
 661                 } else {
 662                     LOG.info("Installing remote McastMac is not executed for this node.");
 663                 }
 664             }
 665
 666             @Override
 667             public void onFailure(Throwable error) {
 668                 LOG.error("Failed to install remote McastMac", error);
 669             }
 670         });
 671     }
 672
 673     private L2GatewayDevice getDeviceFromTunnelIp(String elanInstanceName, IpAddress tunnelIp) {
 674         ConcurrentMap<String, L2GatewayDevice> devices = L2GatewayCacheUtils.getCache();
 675         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 676         for (L2GatewayDevice device : devices.values()) {
 677             if (tunnelIp.equals(device.getTunnelIp())) {
 678                 return device;
 679             }
 680         }
 681         return null;
 682     }
 683
 684     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 685         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 686         if (tunnelInterfaceName == null) {
 687             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 688             return false;
 689         }
 690         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 691                 .Interface tunnelInterface =
 692                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 693         if (tunnelInterface == null) {
 694             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 695             return false;
 696         }
 697         return tunnelInterface.getOperStatus() == OperStatus.Up;
 698     }
 699
 700     public void handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn, List<ListenableFuture<Void>> futures) {
 701         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 702         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 703             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 704                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 705             List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 706             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 707                 LOG.trace("There are no undesignated DPNs");
 708                 return;
 709             }
 710             WriteTransaction tx = broker.newWriteOnlyTransaction();
 711             for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 712                 if (tunnelIp.equals(pair.getLeft())) {
 713                     BigInteger newDesignatedDpn = designateDpnId(tunnelIp, pair.getRight(), dpns);
 714                     if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 715                         Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 716                         if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 717                             LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 718                                     vmMacAddress, newDesignatedDpn);
 719                             installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 720                         }
 721                     }
 722                 }
 723             }
 724             futures.add(tx.submit());
 725         }
 726     }
 727
 728     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 729         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 730         if (tunnelInterfaceName == null) {
 731             return false;
 732         }
 733         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 734         if (tunnelInterface == null) {
 735             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 736             return false;
 737         }
 738         return true;
 739     }
 740
 741     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 742         availableVMCache.remove(tunnelIpElanName);
 743     }
 744
 745     private void unInstallDhcpEntriesOnDpns(final List<BigInteger> dpns, final String vmMacAddress) {
 746         // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 747         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 748                 entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 749                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 750         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 751             @Override
 752             public void onSuccess(Boolean isOwner) {
 753                 if (isOwner) {
 754                     WriteTransaction tx = broker.newWriteOnlyTransaction();
 755                     for (final BigInteger dpn : dpns) {
 756                         unInstallDhcpEntries(dpn, vmMacAddress, tx);
 757                     }
 758                     DhcpServiceUtils.submitTransaction(tx);
 759                 } else {
 760                     LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 761                 }
 762             }
 763
 764             @Override
 765             public void onFailure(Throwable error) {
 766                 LOG.error("Error while fetching checkNodeEntityOwner", error);
 767             }
 768         });
 769     }
 770
 771     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 772         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
 773         IpAddress tunnelIp = null;
 774         Set<Pair<IpAddress, String>> tunnelElanKeySet = availableVMCache.keySet();
 775         Set<String> listExistingVmMacAddress;
 776         for (Pair<IpAddress, String> pair : tunnelElanKeySet) {
 777             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
 778                     pair.getRight());
 779             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 780                 listExistingVmMacAddress = availableVMCache.get(pair);
 781                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
 782                         && listExistingVmMacAddress.contains(vmMacAddress)) {
 783                     tunnelIp = pair.getLeft();
 784                     break;
 785                 }
 786             }
 787         }
 788         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
 789         return tunnelIp;
 790     }
 791 }