vpnservice/dhcpservice/dhcpservice-impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.FutureCallback;
  12 import com.google.common.util.concurrent.Futures;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14
  15 import java.math.BigInteger;
  16 import java.util.ArrayList;
  17 import java.util.Collection;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Set;
  21 import java.util.concurrent.ConcurrentHashMap;
  22 import java.util.concurrent.ConcurrentMap;
  23 import java.util.concurrent.CopyOnWriteArraySet;
  24 import java.util.concurrent.ExecutionException;
  25 import java.util.concurrent.Future;
  26
  27 import org.apache.commons.lang3.tuple.ImmutablePair;
  28 import org.apache.commons.lang3.tuple.Pair;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  31 import org.opendaylight.controller.md.sal.common.api.clustering.EntityOwnershipService;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  34 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  35 import org.opendaylight.genius.mdsalutil.NwConstants;
  36 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  37 import org.opendaylight.genius.utils.clustering.ClusteringUtils;
  38 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  39 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  40 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  41 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  42 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  43 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  44 import org.opendaylight.netvirt.neutronvpn.api.l2gw.utils.L2GatewayCacheUtils;
  45 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  46 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  47 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  50 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorAugmentation;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  73 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  74 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  75 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  76 import org.opendaylight.yangtools.yang.common.RpcResult;
  77 import org.slf4j.Logger;
  78 import org.slf4j.LoggerFactory;
  79
  80 public class DhcpExternalTunnelManager {
  81
  82     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
  83     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
  84
  85     private final DataBroker broker;
  86     private final IMdsalApiManager mdsalUtil;
  87     private final ItmRpcService itmRpcService;
  88     private final EntityOwnershipService entityOwnershipService;
  89     private final IInterfaceManager interfaceManager;
  90
  91     private final ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache = new ConcurrentHashMap<>();
  92     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache = new ConcurrentHashMap<>();
  93     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
  94     private final ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
  95
  96
  97     public DhcpExternalTunnelManager(final DataBroker broker,
  98             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
  99             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager) {
 100         this.broker = broker;
 101         this.mdsalUtil = mdsalUtil;
 102         this.itmRpcService = itmRpcService;
 103         this.entityOwnershipService = entityOwnershipService;
 104         this.interfaceManager = interfaceManager;
 105     }
 106
 107     public void init() {
 108         initilizeCaches();
 109     }
 110
 111     private void initilizeCaches() {
 112         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 113         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 114         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 115         if (designatedSwitchForTunnelOptional.isPresent()) {
 116             List<DesignatedSwitchForTunnel> list = designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 117             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 118                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair = designatedDpnsToTunnelIpElanNameCache.get(designatedSwitchForTunnel.getDpId());
 119                 if (setOfTunnelIpElanNamePair == null) {
 120                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 121                 }
 122                 Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(), designatedSwitchForTunnel.getElanInstanceName());
 123                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 124                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()), setOfTunnelIpElanNamePair);
 125             }
 126         }
 127         LOG.trace("Loading vniMacAddressToPortCache");
 128         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 129         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 130         if (optionalPorts.isPresent()) {
 131             List<Port> list = optionalPorts.get().getPort();
 132             for (Port port : list) {
 133                 if(NeutronUtils.isPortVnicTypeNormal(port)) {
 134                     continue;
 135                 }
 136                 String macAddress = port.getMacAddress().getValue();
 137                 Uuid networkId = port.getNetworkId();
 138                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 139                 if (segmentationId == null) {
 140                     return;
 141                 }
 142                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 143             }
 144         }
 145     }
 146
 147     public BigInteger designateDpnId(IpAddress tunnelIp,
 148             String elanInstanceName, List<BigInteger> dpns) {
 149         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 150         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 151             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 152                     elanInstanceName);
 153             return designatedDpnId;
 154         }
 155         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 156     }
 157
 158     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<BigInteger> dpns,
 159             final BigInteger designatedDpnId, final String vmMacAddress) {
 160         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 161                 elanInstanceName, designatedDpnId, vmMacAddress);
 162         // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 163         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 164                 entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 165                 HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 166         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 167             @Override
 168             public void onSuccess(Boolean isOwner) {
 169                 if (isOwner) {
 170                     synchronized (getTunnelIpDpnKey(tunnelIp, designatedDpnId)) {
 171                         WriteTransaction tx = broker.newWriteOnlyTransaction();
 172                         dpns.remove(designatedDpnId);
 173                         for (BigInteger dpn : dpns) {
 174                             installDhcpDropAction(dpn, vmMacAddress, tx);
 175                         }
 176                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 177                         DhcpServiceUtils.submitTransaction(tx);
 178                     }
 179                 } else {
 180                     LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 181                 }
 182             }
 183
 184             @Override
 185             public void onFailure(Throwable error) {
 186                 LOG.error("Error while fetching checkNodeEntityOwner", error);
 187             }
 188         });
 189         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 190     }
 191
 192     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress, WriteTransaction tx) {
 193         for (String vmMacAddress : listVmMacAddress) {
 194             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 195         }
 196     }
 197
 198     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 199         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 200         removeFromLocalCache(elanInstanceName, vmMacAddress);
 201     }
 202
 203     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 204         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 205             return null;
 206         }
 207         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 208         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 209         if (designatedSwitchForTunnelOptional.isPresent()) {
 210             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 211         }
 212         return null;
 213     }
 214
 215     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 216         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 217         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 218         DesignatedSwitchForTunnel designatedSwitchForTunnel = new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue()).setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp).setKey(designatedSwitchForTunnelKey).build();
 219         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 220                 dpnId);
 221         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 222         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 223     }
 224
 225     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp, String elanInstanceName) {
 226         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey = new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 227         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier = InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 228         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName, dpnId);
 229         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 230         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 231     }
 232
 233     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 234         List<String> vmMacs = getAllVmMacs();
 235         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 236         WriteTransaction tx = broker.newWriteOnlyTransaction();
 237         for (String vmMacAddress : vmMacs) {
 238             installDhcpDropAction(dpId, vmMacAddress, tx);
 239         }
 240         DhcpServiceUtils.submitTransaction(tx);
 241     }
 242
 243     private List<String> getAllVmMacs() {
 244         List<String> vmMacs = new LinkedList<>();
 245         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 246         for (Set<String> list : listOfVmMacs) {
 247             vmMacs.addAll(list);
 248         }
 249         return vmMacs;
 250     }
 251
 252     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 253         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 254         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 255         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 256         if (tunnelIpElanNameSet == null) {
 257             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 258         }
 259         tunnelIpElanNameSet.add(tunnelIpElanName);
 260         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 261                 tunnelIpElanNameSet);
 262         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 263     }
 264
 265     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 266         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 267         Set<String> setOfExistingVmMacAddress;
 268         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 269         if (setOfExistingVmMacAddress == null) {
 270             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 271         }
 272         setOfExistingVmMacAddress.add(vmMacAddress);
 273         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 274                 setOfExistingVmMacAddress);
 275         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 276         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 277     }
 278
 279     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 280         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<IpAddress, String>(tunnelIp, elanInstanceName);
 281         Set<String> listExistingVmMacAddress;
 282         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 283         if (listExistingVmMacAddress == null) {
 284             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 285         }
 286         listExistingVmMacAddress.add(vmMacAddress);
 287         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 288                 listExistingVmMacAddress);
 289         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 290     }
 291
 292     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 293         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 294         try {
 295             Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 296             WriteTransaction tx = broker.newWriteOnlyTransaction();
 297             if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 298                 List<String> listOfVms = getAllVmMacs();
 299                 for (String vmMacAddress : listOfVms) {
 300                     unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 301                 }
 302             }
 303             if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 304                 LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 305                 return;
 306             }
 307             for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 308                 updateCacheAndInstallNewFlows(dpnId, listOfDpns, pair, tx);
 309             }
 310             DhcpServiceUtils.submitTransaction(tx);
 311         } catch (Exception e) {
 312             LOG.error("Error in handleDesignatedDpnDown {}", e);
 313         }
 314     }
 315
 316     public void updateCacheAndInstallNewFlows(BigInteger dpnId,
 317             List<BigInteger> listOfDpns, Pair<IpAddress, String> pair, WriteTransaction tx)
 318             throws ExecutionException {
 319         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 320         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 321             return;
 322         }
 323         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 324         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 325             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 326             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 327         }
 328     }
 329
 330     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId, WriteTransaction tx) {
 331         try {
 332             Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 333             if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 334                 return;
 335             }
 336             for (String vmMacAddress : setOfVmMacAddress) {
 337                 installDhcpDropAction(dpnId, vmMacAddress, tx);
 338             }
 339         } catch (Exception e) {
 340             LOG.error("Error in uninstallExistingFlows {}", e);
 341         }
 342     }
 343
 344     /**
 345      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 346      * @param tunnelIp
 347      * @param elanInstanceName
 348      * @param dpns
 349      * @return
 350      */
 351     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 352             List<BigInteger> dpns) {
 353         BigInteger designatedDpnId = DhcpMConstants.INVALID_DPID;
 354         if (dpns != null && dpns.size() != 0) {
 355             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 356             candidateDpns.retainAll(dpns);
 357             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}", tunnelIp, elanInstanceName, candidateDpns);
 358             boolean elanDpnAvailableFlag = true;
 359             if (candidateDpns == null || candidateDpns.isEmpty()) {
 360                 candidateDpns = dpns;
 361                 elanDpnAvailableFlag = false;
 362             }
 363             int size = 0;
 364             L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 365             if (device == null) {
 366                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}", elanInstanceName, tunnelIp);
 367                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 368                 return designatedDpnId;
 369             }
 370             for (BigInteger dpn : candidateDpns) {
 371                 String hwvtepNodeId = device.getHwvtepNodeId();
 372                 if (!elanDpnAvailableFlag) {
 373                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 374                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 375                         continue;
 376                     }
 377                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 378                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 379                     continue;
 380                 }
 381                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 382                 if (tunnelIpElanNameSet == null) {
 383                     designatedDpnId = dpn;
 384                     break;
 385                 }
 386                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 387                     size = tunnelIpElanNameSet.size();
 388                     designatedDpnId = dpn;
 389                 }
 390             }
 391             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 392             return designatedDpnId;
 393         }
 394         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 395         return designatedDpnId;
 396     }
 397
 398     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 399         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 400     }
 401
 402     private void installDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 403         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 404     }
 405
 406     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 407         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, tx);
 408     }
 409
 410     private void installDhcpDropAction(BigInteger dpn, String vmMacAddress, WriteTransaction tx) {
 411         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL, vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 412     }
 413
 414     public void handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn, List<ListenableFuture<Void>> futures) {
 415         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 416         if (interfaceDpn == null) {
 417             return;
 418         }
 419         try {
 420             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 421                 Set<Pair<IpAddress, String>> tunnelElanPairSet = designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 422                 if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 423                     return;
 424                 }
 425                 WriteTransaction tx = broker.newWriteOnlyTransaction();
 426                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 427                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 428                     if (tunnelIpInDpn.equals(tunnelIp)) {
 429                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 430                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 431                                     tunnelElanPair);
 432                             return;
 433                         }
 434                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 435                         dpns.remove(interfaceDpn);
 436                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 437                         updateCacheAndInstallNewFlows(interfaceDpn, dpns, tunnelElanPair, tx);
 438                     }
 439                 }
 440                 futures.add(tx.submit());
 441             }
 442         } catch (Exception e) {
 443             LOG.error("Error in handleTunnelStateDown {}", e.getMessage());
 444             LOG.trace("Exception details {}", e);
 445         }
 446     }
 447
 448     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 449         ConcurrentMap<String, L2GatewayDevice> l2GwDevices = ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 450         for (L2GatewayDevice device : l2GwDevices.values()) {
 451             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 452                 return true;
 453             }
 454         }
 455         return false;
 456     }
 457
 458     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 459         return tunnelIp.toString() + interfaceDpn;
 460     }
 461
 462     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 463         Set<Pair<IpAddress, String>> tunnelIpElanNameKeySet = tunnelIpElanNameToVmMacCache.keySet();
 464         for (Pair<IpAddress, String> pair : tunnelIpElanNameKeySet) {
 465             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 466                 Set<String> setOfExistingVmMacAddress;
 467                 setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 468                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 469                     continue;
 470                 }
 471                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 472                         setOfExistingVmMacAddress, elanInstanceName);
 473                 setOfExistingVmMacAddress.remove(vmMacAddress);
 474                 if (setOfExistingVmMacAddress.size() > 0) {
 475                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 476                     return;
 477                 }
 478                 tunnelIpElanNameToVmMacCache.remove(pair);
 479             }
 480         }
 481     }
 482
 483     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 484         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 485         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 486         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 487         if (tunnelIpElanNameSet != null) {
 488             LOG.trace(
 489                     "Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for " +
 490                             "designatedDpnId {}",
 491                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 492             tunnelIpElanNameSet.remove(tunnelIpElanName);
 493             if (tunnelIpElanNameSet.size() != 0) {
 494                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 495             } else {
 496                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 497             }
 498         }
 499     }
 500
 501     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 502         if (macAddress == null) {
 503             return;
 504         }
 505         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 506         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 507                 macAddress.toUpperCase(), vniMacAddressPair, port);
 508         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 509     }
 510
 511     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 512         if (macAddress == null) {
 513             return;
 514         }
 515         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 516         vniMacAddressToPortCache.remove(vniMacAddressPair);
 517     }
 518
 519     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 520         if (macAddress == null) {
 521             return null;
 522         }
 523         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(vni, macAddress.toUpperCase());
 524         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni, macAddress.toUpperCase(), vniMacAddressPair, vniMacAddressToPortCache.get(vniMacAddressPair));
 525         return vniMacAddressToPortCache.get(vniMacAddressPair);
 526     }
 527
 528     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 529         String tunnelInterfaceName = null;
 530         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 531         try {
 532             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 533                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 534                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 535
 536             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 537             if (rpcResult.isSuccessful()) {
 538                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 539                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 540             } else {
 541                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 542             }
 543         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 544             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}: {} ",
 545                     sourceNode, dstNode, e);
 546         }
 547         return tunnelInterfaceName;
 548     }
 549
 550     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 551         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 552                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 553         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 554     }
 555
 556     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 557         List<LocatorSet> locators = new ArrayList<>();
 558         for (TerminationPoint tp : dstDevice.getTerminationPoint()) {
 559             HwvtepPhysicalLocatorAugmentation aug = tp.getAugmentation(HwvtepPhysicalLocatorAugmentation.class);
 560             if (internalTunnelIp.getIpv4Address().equals(aug.getDstIp().getIpv4Address())) {
 561                 HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 562                         HwvtepSouthboundUtils.createPhysicalLocatorInstanceIdentifier(dstDevice.getNodeId(), aug));
 563                 locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 564             }
 565         }
 566         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 567                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 568
 569         RemoteMcastMacs remoteUcastMacs = new RemoteMcastMacsBuilder()
 570                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 571                 .setLogicalSwitchRef(lsRef).setLocatorSet(locators).build();
 572         return remoteUcastMacs;
 573     }
 574
 575     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName, L2GatewayDevice device, IpAddress internalTunnelIp) {
 576         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 577         Node dstNode = optionalNode.get();
 578         if (dstNode == null) {
 579             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 580             return null;
 581         }
 582         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 583         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 584         return transaction;
 585     }
 586
 587     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp, final String elanInstanceName) {
 588         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 589             return;
 590         }
 591         ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE, HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 592         Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 593             @Override
 594             public void onSuccess(Boolean isOwner) {
 595                 if (isOwner) {
 596                     LOG.info("Installing remote McastMac");
 597                     L2GatewayDevice device = getDeviceFromTunnelIp(elanInstanceName, tunnelIp);
 598                     String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId), device.getHwvtepNodeId());
 599                     IpAddress internalTunnelIp = null;
 600                     if (tunnelInterfaceName != null) {
 601                         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 602                         if (tunnelInterface == null) {
 603                             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 604                             return;
 605                         }
 606                         internalTunnelIp = tunnelInterface.getAugmentation(IfTunnel.class).getTunnelSource();
 607                         WriteTransaction transaction = broker.newWriteOnlyTransaction();
 608                         putRemoteMcastMac(transaction, elanInstanceName, device, internalTunnelIp);
 609                         if (transaction != null) {
 610                             transaction.submit();
 611                         }
 612                     }
 613                 } else {
 614                     LOG.info("Installing remote McastMac is not executed for this node.");
 615                 }
 616             }
 617
 618             @Override
 619             public void onFailure(Throwable error) {
 620                 LOG.error("Failed to install remote McastMac", error);
 621             }
 622         });
 623     }
 624
 625     private L2GatewayDevice getDeviceFromTunnelIp(String elanInstanceName, IpAddress tunnelIp) {
 626         ConcurrentMap<String, L2GatewayDevice> devices = L2GatewayCacheUtils.getCache();
 627         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 628         for (L2GatewayDevice device : devices.values()) {
 629             if (tunnelIp.equals(device.getTunnelIp())) {
 630                 return device;
 631             }
 632         }
 633         return null;
 634     }
 635
 636     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 637         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 638         if (tunnelInterfaceName == null) {
 639             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 640             return false;
 641         }
 642         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface tunnelInterface = DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 643         if (tunnelInterface == null) {
 644             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 645             return false;
 646         }
 647         return tunnelInterface.getOperStatus() == OperStatus.Up;
 648     }
 649
 650     public void handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn, List<ListenableFuture<Void>> futures) {
 651         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 652         try {
 653             synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 654                 Set<Pair<IpAddress, String>> tunnelIpElanPair = designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 655                 List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 656                 if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 657                     LOG.trace("There are no undesignated DPNs");
 658                     return;
 659                 }
 660                 WriteTransaction tx = broker.newWriteOnlyTransaction();
 661                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 662                     if (tunnelIp.equals(pair.getLeft())) {
 663                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, pair.getRight(), dpns);
 664                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 665                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 666                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 667                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}", vmMacAddress, newDesignatedDpn);
 668                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 669                             }
 670                         }
 671                     }
 672                 }
 673                 futures.add(tx.submit());
 674             }
 675         } catch (Exception e) {
 676             LOG.error("Error in handleTunnelStateUp {}", e.getMessage());
 677             LOG.trace("Exception details {}", e);
 678         }
 679     }
 680
 681     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 682         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 683         if (tunnelInterfaceName == null) {
 684             return false;
 685         }
 686         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 687         if (tunnelInterface == null) {
 688             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 689             return false;
 690         }
 691         return true;
 692     }
 693
 694     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 695         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 696         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 697         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 698                 elanInstanceName, tunnelIp, dpns, vmMacs);
 699         if (vmMacs == null) {
 700             return;
 701         }
 702         for (String vmMacAddress : vmMacs) {
 703             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 704         }
 705         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 706     }
 707
 708     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 709         availableVMCache.remove(tunnelIpElanName);
 710     }
 711
 712     private void unInstallDhcpEntriesOnDpns(final List<BigInteger> dpns, final String vmMacAddress) {
 713             // TODO: Make use a util that directly tells if this is the owner or not rather than making use of callbacks.
 714             ListenableFuture<Boolean> checkEntityOwnerFuture = ClusteringUtils.checkNodeEntityOwner(
 715                     entityOwnershipService, HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 716                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME);
 717             Futures.addCallback(checkEntityOwnerFuture, new FutureCallback<Boolean>() {
 718                 @Override
 719                 public void onSuccess(Boolean isOwner) {
 720                     if (isOwner) {
 721                         WriteTransaction tx = broker.newWriteOnlyTransaction();
 722                         for (final BigInteger dpn : dpns) {
 723                             unInstallDhcpEntries(dpn, vmMacAddress, tx);
 724                         }
 725                         DhcpServiceUtils.submitTransaction(tx);
 726                     } else {
 727                         LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 728                     }
 729                 }
 730
 731                 @Override
 732                 public void onFailure(Throwable error) {
 733                     LOG.error("Error while fetching checkNodeEntityOwner", error);
 734                 }
 735             });
 736     }
 737
 738     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 739         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
 740         IpAddress tunnelIp = null;
 741         Set<Pair<IpAddress, String>> tunnelElanKeySet = availableVMCache.keySet();
 742         Set<String> listExistingVmMacAddress;
 743         for (Pair<IpAddress, String> pair : tunnelElanKeySet) {
 744             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
 745                     pair.getRight());
 746             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 747                 listExistingVmMacAddress = availableVMCache.get(pair);
 748                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty() &&
 749                         listExistingVmMacAddress.contains(vmMacAddress)) {
 750                     tunnelIp = pair.getLeft();
 751                     break;
 752                 }
 753             }
 754         }
 755         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
 756         return tunnelIp;
 757     }
 758 }