dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ListenableFuture;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.HashSet;
  17 import java.util.LinkedList;
  18 import java.util.List;
  19 import java.util.Locale;
  20 import java.util.Map.Entry;
  21 import java.util.Set;
  22 import java.util.concurrent.ConcurrentHashMap;
  23 import java.util.concurrent.ConcurrentMap;
  24 import java.util.concurrent.CopyOnWriteArraySet;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.annotation.PostConstruct;
  28 import javax.inject.Inject;
  29 import javax.inject.Named;
  30 import javax.inject.Singleton;
  31 import org.apache.commons.lang3.tuple.ImmutablePair;
  32 import org.apache.commons.lang3.tuple.Pair;
  33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  34 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  45 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  46 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  47 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  50 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  51 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  53 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  56 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  57 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  58 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  89 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  90 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  91 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
  92 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  93 import org.opendaylight.yangtools.yang.common.RpcResult;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 @Singleton
  98 public class DhcpExternalTunnelManager {
  99
 100     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 101     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 102
 103     private final DataBroker broker;
 104     private final ManagedNewTransactionRunner txRunner;
 105     private final IMdsalApiManager mdsalUtil;
 106     private final ItmRpcService itmRpcService;
 107     private final EntityOwnershipUtils entityOwnershipUtils;
 108     private final IInterfaceManager interfaceManager;
 109     private final JobCoordinator jobCoordinator;
 110     private final L2GatewayCache l2GatewayCache;
 111     private IElanService elanService;
 112
 113     private final ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 114             new ConcurrentHashMap<>();
 115     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 116             new ConcurrentHashMap<>();
 117     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 118     private final ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 119
 120     @Inject
 121     public DhcpExternalTunnelManager(final DataBroker broker,
 122             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 123             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 124             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 125             @Named("elanService") IElanService ielanService) {
 126         this.broker = broker;
 127         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 128         this.mdsalUtil = mdsalUtil;
 129         this.itmRpcService = itmRpcService;
 130         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 131         this.interfaceManager = interfaceManager;
 132         this.jobCoordinator = jobCoordinator;
 133         this.l2GatewayCache = l2GatewayCache;
 134         this.elanService = ielanService;
 135     }
 136
 137     @PostConstruct
 138     public void init() {
 139         initilizeCaches();
 140     }
 141
 142     private void initilizeCaches() {
 143         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 144         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 145                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 146         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 147                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 148         if (designatedSwitchForTunnelOptional.isPresent()) {
 149             List<DesignatedSwitchForTunnel> list =
 150                     designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 151             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 152                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 153                         designatedDpnsToTunnelIpElanNameCache
 154                                 .get(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()));
 155                 if (setOfTunnelIpElanNamePair == null) {
 156                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 157                 }
 158                 Pair<IpAddress, String> tunnelIpElanNamePair =
 159                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 160                                 designatedSwitchForTunnel.getElanInstanceName());
 161                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 162                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()),
 163                         setOfTunnelIpElanNamePair);
 164             }
 165         }
 166         LOG.trace("Loading vniMacAddressToPortCache");
 167         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 168         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 169         if (optionalPorts.isPresent()) {
 170             List<Port> list = optionalPorts.get().getPort();
 171             for (Port port : list) {
 172                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 173                     continue;
 174                 }
 175                 String macAddress = port.getMacAddress().getValue();
 176                 Uuid networkId = port.getNetworkId();
 177                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 178                 if (segmentationId == null) {
 179                     return;
 180                 }
 181                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 182             }
 183         }
 184     }
 185
 186     public BigInteger designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns) {
 187         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 188         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 189             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 190                     elanInstanceName);
 191             return designatedDpnId;
 192         }
 193         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 194     }
 195
 196     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<BigInteger> dpns,
 197             final BigInteger designatedDpnId, final String vmMacAddress) {
 198         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 199                 elanInstanceName, designatedDpnId, vmMacAddress);
 200
 201         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 202         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 203             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 204                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 205                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 206                     dpns.remove(designatedDpnId);
 207                     for (BigInteger dpn : dpns) {
 208                         installDhcpDropAction(dpn, vmMacAddress, tx);
 209                     }
 210                     installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 211                 }));
 212             } else {
 213                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 214             }
 215
 216             return Collections.emptyList();
 217         });
 218
 219         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 220     }
 221
 222     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress, WriteTransaction tx) {
 223         for (String vmMacAddress : listVmMacAddress) {
 224             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 225         }
 226     }
 227
 228     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 229         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 230         removeFromLocalCache(elanInstanceName, vmMacAddress);
 231     }
 232
 233     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 234         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 235         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 236         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 237                 elanInstanceName, tunnelIp, dpns, vmMacs);
 238         if (vmMacs == null) {
 239             return;
 240         }
 241         for (String vmMacAddress : vmMacs) {
 242             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 243         }
 244         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 245     }
 246
 247     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 248         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 249             return null;
 250         }
 251         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 252                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 253                         .child(DesignatedSwitchForTunnel.class,
 254                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 255         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional =
 256                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 257         if (designatedSwitchForTunnelOptional.isPresent()) {
 258             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 259         }
 260         return null;
 261     }
 262
 263     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 264                                                        String elanInstanceName) {
 265         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 266                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 267         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 268                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 269                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 270         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 271                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 272                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 273                         .setKey(designatedSwitchForTunnelKey).build();
 274         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 275                 dpnId);
 276         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 277         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 278     }
 279
 280     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 281                                                         String elanInstanceName) {
 282         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 283                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 284         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 285                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 286                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 287         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 288                 elanInstanceName, dpnId);
 289         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 290         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 291     }
 292
 293     // This method is called whenever new OVS Switch is added.
 294     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 295         // During controller restart we'll get add for designatedDpns as well and we
 296         // need not install drop flows for those dpns
 297         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 298             LOG.trace("The dpn {} is designated DPN need not install drop flows");
 299             return;
 300         }
 301         // Read from DS since the cache may not get loaded completely in restart scenario
 302         if (isDpnDesignatedDpn(dpId)) {
 303             LOG.trace("The dpn {} is designated DPN need not install drop flows");
 304             return;
 305         }
 306         List<String> vmMacs = getAllVmMacs();
 307         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 308         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 309             for (String vmMacAddress : vmMacs) {
 310                 installDhcpDropAction(dpId, vmMacAddress, tx);
 311             }
 312         }), LOG, "Error writing to the datastore");
 313     }
 314
 315     private boolean isDpnDesignatedDpn(BigInteger dpId) {
 316         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 317                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 318         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 319                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 320         if (designatedSwitchForTunnelOptional.isPresent()) {
 321             List<DesignatedSwitchForTunnel> list =
 322                     designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 323             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 324                 if (dpId.equals(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()))) {
 325                     return true;
 326                 }
 327             }
 328         }
 329         return false;
 330     }
 331
 332     private List<String> getAllVmMacs() {
 333         List<String> vmMacs = new LinkedList<>();
 334         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 335         for (Set<String> list : listOfVmMacs) {
 336             vmMacs.addAll(list);
 337         }
 338         return vmMacs;
 339     }
 340
 341     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 342         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 343         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 344         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 345         if (tunnelIpElanNameSet == null) {
 346             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 347         }
 348         tunnelIpElanNameSet.add(tunnelIpElanName);
 349         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 350                 tunnelIpElanNameSet);
 351         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 352     }
 353
 354     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 355         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 356         Set<String> setOfExistingVmMacAddress;
 357         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 358         if (setOfExistingVmMacAddress == null) {
 359             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 360         }
 361         setOfExistingVmMacAddress.add(vmMacAddress);
 362         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 363                 setOfExistingVmMacAddress);
 364         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 365         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 366     }
 367
 368     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 369         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 370         Set<String> listExistingVmMacAddress;
 371         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 372         if (listExistingVmMacAddress == null) {
 373             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 374         }
 375         listExistingVmMacAddress.add(vmMacAddress);
 376         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 377                 listExistingVmMacAddress);
 378         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 379     }
 380
 381     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 382         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 383         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 384         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 385             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 386         } else {
 387             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 388                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 389                     List<String> listOfVms = getAllVmMacs();
 390                     for (String vmMacAddress : listOfVms) {
 391                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 392                     }
 393                 }
 394                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 395                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 396                 }
 397             }), LOG, "Error writing to datastore");
 398         }
 399     }
 400
 401     public void updateCacheAndInstallNewFlows(List<BigInteger> listOfDpns, Pair<IpAddress, String> pair,
 402             WriteTransaction tx) {
 403         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 404         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 405             return;
 406         }
 407         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 408         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 409             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 410             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 411         }
 412         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 413         if (subnetDhcpData.isPresent()) {
 414             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 415                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 416         }
 417     }
 418
 419     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId,
 420                                           WriteTransaction tx) {
 421         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 422         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 423             return;
 424         }
 425         for (String vmMacAddress : setOfVmMacAddress) {
 426             installDhcpDropAction(dpnId, vmMacAddress, tx);
 427         }
 428     }
 429
 430     /**
 431      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 432      * @param tunnelIp The tunnel Ip address
 433      * @param elanInstanceName The elan instance name
 434      * @param dpns The data path nodes
 435      * @return The designated dpn
 436      */
 437     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 438             List<BigInteger> dpns) {
 439         BigInteger designatedDpnId = DhcpMConstants.INVALID_DPID;
 440         if (dpns != null && dpns.size() != 0) {
 441             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 442             candidateDpns.retainAll(dpns);
 443             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 444                     tunnelIp, elanInstanceName, candidateDpns);
 445             boolean elanDpnAvailableFlag = true;
 446             if (candidateDpns.isEmpty()) {
 447                 candidateDpns = dpns;
 448                 elanDpnAvailableFlag = false;
 449             }
 450             int size = 0;
 451             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 452             if (device == null) {
 453                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 454                         elanInstanceName, tunnelIp);
 455                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 456                 return designatedDpnId;
 457             }
 458             for (BigInteger dpn : candidateDpns) {
 459                 String hwvtepNodeId = device.getHwvtepNodeId();
 460                 if (!elanDpnAvailableFlag) {
 461                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 462                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 463                         continue;
 464                     }
 465                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 466                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 467                     continue;
 468                 }
 469                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 470                 if (tunnelIpElanNameSet == null) {
 471                     designatedDpnId = dpn;
 472                     break;
 473                 }
 474                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 475                     size = tunnelIpElanNameSet.size();
 476                     designatedDpnId = dpn;
 477                 }
 478             }
 479             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 480             return designatedDpnId;
 481         }
 482         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 483         return designatedDpnId;
 484     }
 485
 486     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 487         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 488     }
 489
 490     private void installDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 491         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 492                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 493     }
 494
 495     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 496                                               String dhcpMacAddress) {
 497         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 498         for (Entry<BigInteger, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 499             BigInteger dpn = entry.getKey();
 500             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 501             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 502                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 503                     if (addFlow) {
 504                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 505                                 elanInstanceName, pair.getLeft());
 506                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 507                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 508                     } else {
 509                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 510                                 elanInstanceName, pair.getLeft());
 511                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 512                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 513                     }
 514                 }
 515             }
 516         }
 517     }
 518
 519
 520     public void configureDhcpArpRequestResponseFlow(BigInteger dpnId, String elanInstanceName, boolean addFlow,
 521                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 522         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 523         if (device == null) {
 524             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 525                     elanInstanceName);
 526         }
 527         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 528             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 529                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 530                 String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpnId),
 531                         device.getHwvtepNodeId());
 532                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 533                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 534                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 535                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 536                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 537                 if (optElan.isPresent()) {
 538                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 539                             lportTag, optElan.get().getElanTag());
 540                     Uuid nwUuid = new Uuid(elanInstanceName);
 541                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 542                     BigInteger vni = strVni != null ? new BigInteger(strVni) : BigInteger.ZERO;
 543                     if (!vni.equals(BigInteger.ZERO)) {
 544                         if (addFlow) {
 545                             LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 546                                     dpnId, dhcpIpAddress, lportTag);
 547                             installDhcpArpRequestFlows(dpnId, vni, dhcpIpAddress, lportTag,
 548                                     optElan.get().getElanTag());
 549                             installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 550                                     dhcpIpAddress, dhcpMacAddress);
 551                         } else {
 552                             LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 553                                     dpnId, dhcpIpAddress, lportTag);
 554                             uninstallDhcpArpRequestFlows(dpnId, vni, dhcpIpAddress, lportTag);
 555                             uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 556                         }
 557                     }
 558                 }
 559             }
 560             return null;
 561         });
 562     }
 563
 564     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 565         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 566         Uuid nwUuid = new Uuid(elanInstanceName);
 567         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 568         for (Uuid subnet : subnets) {
 569             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 570                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 571                 return optSubnetDhcp;
 572             }
 573         }
 574         return optSubnetDhcp;
 575     }
 576
 577     private void installDhcpArpRequestFlows(BigInteger dpnId, BigInteger vni, String dhcpIpAddress,
 578                                             int lportTag, Long elanTag) {
 579         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 580                 lportTag, elanTag, true, mdsalUtil);
 581     }
 582
 583     private void installDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 584                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 585         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 586                 elanInstanceName, lportTag, dhcpIpAddress);
 587         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 588         builder.setDpId(dpnId).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 589                 .setLportTag(lportTag);
 590         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 591                 dhcpIpAddress, dhcpMacAddress));
 592         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 593     }
 594
 595     private void uninstallDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 596                                                 String dhcpIpAddress) {
 597         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 598         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId)
 599                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 600                 .setLportTag(lportTag).buildForRemoveFlow();
 601         elanService.removeArpResponderFlow(arpInput);
 602     }
 603
 604     private void uninstallDhcpArpRequestFlows(BigInteger dpnId, BigInteger vni, String dhcpIpAddress,
 605                                               int lportTag) {
 606         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 607                 lportTag, null, false, mdsalUtil);
 608     }
 609
 610
 611     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 612         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 613                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, tx);
 614     }
 615
 616     private void installDhcpDropAction(BigInteger dpn, String vmMacAddress, WriteTransaction tx) {
 617         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 618                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 619     }
 620
 621     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn) {
 622         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 623         if (interfaceDpn == null) {
 624             return Collections.emptyList();
 625         }
 626         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 627             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 628                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 629             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 630                 return Collections.emptyList();
 631             }
 632             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 633                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 634                 if (tunnelIpInDpn.equals(tunnelIp)) {
 635                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 636                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 637                                 tunnelElanPair);
 638                         return Collections.emptyList();
 639                     }
 640                 }
 641             }
 642             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 643                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 644                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 645                     String elanInstanceName = tunnelElanPair.getRight();
 646                     if (tunnelIpInDpn.equals(tunnelIp)) {
 647                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 648                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 649                                     tunnelElanPair);
 650                         }
 651                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 652                         dpns.remove(interfaceDpn);
 653                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 654                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 655                         if (subnetDhcpData.isPresent()) {
 656                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 657                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 658                                     subnetDhcpData.get().getPortMacaddress());
 659                         }
 660                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 661                     }
 662                 }
 663             }));
 664         }
 665     }
 666
 667     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 668         ConcurrentMap<String, L2GatewayDevice> l2GwDevices =
 669                 ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 670         for (L2GatewayDevice device : l2GwDevices.values()) {
 671             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 672                 return true;
 673             }
 674         }
 675         return false;
 676     }
 677
 678     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 679         return tunnelIp.toString() + interfaceDpn;
 680     }
 681
 682     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 683         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 684             Pair<IpAddress, String> pair = entry.getKey();
 685             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 686                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 687                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 688                     continue;
 689                 }
 690                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 691                         setOfExistingVmMacAddress, elanInstanceName);
 692                 setOfExistingVmMacAddress.remove(vmMacAddress);
 693                 if (setOfExistingVmMacAddress.size() > 0) {
 694                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 695                     return;
 696                 }
 697                 tunnelIpElanNameToVmMacCache.remove(pair);
 698             }
 699         }
 700     }
 701
 702     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 703         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 704         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 705         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 706         if (tunnelIpElanNameSet != null) {
 707             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 708                             + "designatedDpnId {}",
 709                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 710             tunnelIpElanNameSet.remove(tunnelIpElanName);
 711             if (tunnelIpElanNameSet.size() != 0) {
 712                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 713             } else {
 714                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 715             }
 716         }
 717     }
 718
 719     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 720         if (macAddress == null) {
 721             return;
 722         }
 723         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 724                 vni, macAddress.toUpperCase(Locale.getDefault()));
 725         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 726                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 727         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 728     }
 729
 730     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 731         if (macAddress == null) {
 732             return;
 733         }
 734         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 735                 vni, macAddress.toUpperCase(Locale.getDefault()));
 736         vniMacAddressToPortCache.remove(vniMacAddressPair);
 737     }
 738
 739     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 740         if (macAddress == null) {
 741             return null;
 742         }
 743         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 744                 vni, macAddress.toUpperCase(Locale.getDefault()));
 745         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 746                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 747                 vniMacAddressToPortCache.get(vniMacAddressPair));
 748         return vniMacAddressToPortCache.get(vniMacAddressPair);
 749     }
 750
 751     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 752         String tunnelInterfaceName = null;
 753         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 754         try {
 755             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 756                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 757                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 758
 759             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 760             if (rpcResult.isSuccessful()) {
 761                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 762                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 763             } else {
 764                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 765             }
 766         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 767             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 768                     sourceNode, dstNode, e);
 769         }
 770         return tunnelInterfaceName;
 771     }
 772
 773     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 774         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 775                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 776         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 777     }
 778
 779     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 780         Set<LocatorSet> locators = new HashSet<>();
 781         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 782                 internalTunnelIp.getIpv4Address().getValue());
 783         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 784                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 785                         terminationPointKey));
 786         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 787
 788         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 789                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 790
 791         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 792                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 793                 .setLogicalSwitchRef(lsRef).build();
 794         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 795                 dstDevice.getNodeId(), remoteMcastMacs.getKey());
 796         ReadOnlyTransaction transaction = broker.newReadOnlyTransaction();
 797         try {
 798             //TODO do async mdsal read
 799             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).checkedGet().get();
 800             locators.addAll(remoteMcastMacs.getLocatorSet());
 801             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 802         } catch (ReadFailedException e) {
 803             LOG.error("Failed to read the macs {}", iid);
 804         }
 805         return null;
 806     }
 807
 808     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName,
 809                                                L2GatewayDevice device, IpAddress internalTunnelIp) {
 810         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 811         Node dstNode = optionalNode.get();
 812         if (dstNode == null) {
 813             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 814             return null;
 815         }
 816         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 817         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 818         return transaction;
 819     }
 820
 821     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp,
 822                                       final String elanInstanceName) {
 823         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 824             return;
 825         }
 826
 827         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 828             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 829                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 830                 LOG.info("Installing remote McastMac is not executed for this node.");
 831                 return Collections.emptyList();
 832             }
 833
 834             LOG.info("Installing remote McastMac");
 835             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 836             if (device == null) {
 837                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 838                     elanInstanceName);
 839                 return Collections.emptyList();
 840             }
 841             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 842                     device.getHwvtepNodeId());
 843             if (tunnelInterfaceName != null) {
 844                 Interface tunnelInterface =
 845                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 846                 if (tunnelInterface == null) {
 847                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 848                     return Collections.emptyList();
 849                 }
 850                 return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 851                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 852                             tunnelInterface.getAugmentation(IfTunnel.class).getTunnelSource())));
 853             }
 854             return Collections.emptyList();
 855         });
 856     }
 857
 858     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 859         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 860         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 861         for (L2GatewayDevice device : devices) {
 862             if (tunnelIp.equals(device.getTunnelIp())) {
 863                 return device;
 864             }
 865         }
 866         return null;
 867     }
 868
 869     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 870         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 871         if (tunnelInterfaceName == null) {
 872             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 873             return false;
 874         }
 875         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 876                 .Interface tunnelInterface =
 877                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 878         if (tunnelInterface == null) {
 879             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 880             return false;
 881         }
 882         return tunnelInterface.getOperStatus() == OperStatus.Up;
 883     }
 884
 885     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn) {
 886         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 887         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 888             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 889                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 890             List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 891             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 892                 LOG.trace("There are no undesignated DPNs");
 893                 return Collections.emptyList();
 894             }
 895             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 896                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 897                     if (tunnelIp.equals(pair.getLeft())) {
 898                         String elanInstanceName = pair.getRight();
 899                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 900                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 901                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 902                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 903                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 904                                         vmMacAddress, newDesignatedDpn);
 905                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 906                             }
 907                         }
 908                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 909                         if (subnetDhcpData.isPresent()) {
 910                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 911                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 912                                     subnetDhcpData.get().getPortMacaddress());
 913                         }
 914                     }
 915                 }
 916             }));
 917         }
 918     }
 919
 920     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 921         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 922         if (tunnelInterfaceName == null) {
 923             return false;
 924         }
 925         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 926         if (tunnelInterface == null) {
 927             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 928             return false;
 929         }
 930         return true;
 931     }
 932
 933     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 934         availableVMCache.remove(tunnelIpElanName);
 935     }
 936
 937     private void unInstallDhcpEntriesOnDpns(final List<BigInteger> dpns, final String vmMacAddress) {
 938         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
 939             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 940                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 941                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 942                     for (final BigInteger dpn : dpns) {
 943                         unInstallDhcpEntries(dpn, vmMacAddress, tx);
 944                     }
 945                 }));
 946             } else {
 947                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 948             }
 949
 950             return Collections.emptyList();
 951         });
 952     }
 953
 954     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 955         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
 956         IpAddress tunnelIp = null;
 957         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
 958             Pair<IpAddress, String> pair = entry.getKey();
 959             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
 960                     pair.getRight());
 961             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 962                 Set<String> listExistingVmMacAddress = entry.getValue();
 963                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
 964                         && listExistingVmMacAddress.contains(vmMacAddress)) {
 965                     tunnelIp = pair.getLeft();
 966                     break;
 967                 }
 968             }
 969         }
 970         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
 971         return tunnelIp;
 972     }
 973
 974     private String getJobKey(final String jobKeySuffix) {
 975         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
 976     }
 977 }