dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14 import java.math.BigInteger;
  15 import java.util.ArrayList;
  16 import java.util.Collection;
  17 import java.util.Collections;
  18 import java.util.HashSet;
  19 import java.util.LinkedList;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map.Entry;
  23 import java.util.Objects;
  24 import java.util.Set;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentMap;
  27 import java.util.concurrent.CopyOnWriteArraySet;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import javax.annotation.PostConstruct;
  31 import javax.inject.Inject;
  32 import javax.inject.Named;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.tuple.ImmutablePair;
  35 import org.apache.commons.lang3.tuple.Pair;
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.infra.Datastore;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  46 import org.opendaylight.genius.infra.TypedWriteTransaction;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  51 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  52 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  53 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  54 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  57 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  58 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  59 import org.opendaylight.netvirt.dhcpservice.api.IDhcpExternalTunnelManager;
  60 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  61 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  64 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  65 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  66 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  97 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  98 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  99 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
 100 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 101 import org.opendaylight.yangtools.yang.common.RpcResult;
 102 import org.opendaylight.yangtools.yang.common.Uint64;
 103 import org.slf4j.Logger;
 104 import org.slf4j.LoggerFactory;
 105
 106 @Singleton
 107 public class DhcpExternalTunnelManager implements IDhcpExternalTunnelManager {
 108
 109     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 110     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 111
 112     private final DataBroker broker;
 113     private final ManagedNewTransactionRunner txRunner;
 114     private final IMdsalApiManager mdsalUtil;
 115     private final ItmRpcService itmRpcService;
 116     private final EntityOwnershipUtils entityOwnershipUtils;
 117     private final IInterfaceManager interfaceManager;
 118     private final JobCoordinator jobCoordinator;
 119     private final L2GatewayCache l2GatewayCache;
 120     private IElanService elanService;
 121     private final DhcpServiceCounters dhcpServiceCounters;
 122
 123     private final ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 124             new ConcurrentHashMap<>();
 125     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 126             new ConcurrentHashMap<>();
 127     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 128     private final ConcurrentMap<Pair<Uint64, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 129
 130     @Override
 131     public ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> getDesignatedDpnsToTunnelIpElanNameCache() {
 132         return designatedDpnsToTunnelIpElanNameCache;
 133     }
 134
 135     @Override
 136     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getTunnelIpElanNameToVmMacCache() {
 137         return tunnelIpElanNameToVmMacCache;
 138     }
 139
 140     @Override
 141     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getAvailableVMCache() {
 142         return availableVMCache;
 143     }
 144
 145     @Override
 146     public ConcurrentMap<Pair<Uint64, String>, Port> getVniMacAddressToPortCache() {
 147         return vniMacAddressToPortCache;
 148     }
 149
 150     @Inject
 151     public DhcpExternalTunnelManager(final DataBroker broker,
 152             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 153             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 154             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 155             @Named("elanService") IElanService ielanService, DhcpServiceCounters dhcpServiceCounters) {
 156         this.broker = broker;
 157         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 158         this.mdsalUtil = mdsalUtil;
 159         this.itmRpcService = itmRpcService;
 160         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 161         this.interfaceManager = interfaceManager;
 162         this.jobCoordinator = jobCoordinator;
 163         this.l2GatewayCache = l2GatewayCache;
 164         this.elanService = ielanService;
 165         this.dhcpServiceCounters = dhcpServiceCounters;
 166     }
 167
 168     @PostConstruct
 169     public void init() {
 170         initilizeCaches();
 171     }
 172
 173     private void initilizeCaches() {
 174         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 175         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 176                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 177         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 178                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 179         if (designatedSwitchForTunnelOptional.isPresent()) {
 180             List<DesignatedSwitchForTunnel> list =
 181                 designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 182             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 183                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 184                         designatedDpnsToTunnelIpElanNameCache
 185                                 .get(Uint64.valueOf(designatedSwitchForTunnel.getDpId()));
 186                 if (setOfTunnelIpElanNamePair == null) {
 187                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 188                 }
 189                 Pair<IpAddress, String> tunnelIpElanNamePair =
 190                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 191                                 designatedSwitchForTunnel.getElanInstanceName());
 192                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 193                 designatedDpnsToTunnelIpElanNameCache.put(Uint64.valueOf(designatedSwitchForTunnel.getDpId()),
 194                         setOfTunnelIpElanNamePair);
 195             }
 196         }
 197         LOG.trace("Loading vniMacAddressToPortCache");
 198         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 199         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 200         if (optionalPorts.isPresent()) {
 201             List<Port> list = optionalPorts.get().nonnullPort();
 202             for (Port port : list) {
 203                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 204                     continue;
 205                 }
 206                 String macAddress = port.getMacAddress().getValue();
 207                 Uuid networkId = port.getNetworkId();
 208                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 209                 if (segmentationId == null) {
 210                     return;
 211                 }
 212                 updateVniMacToPortCache(Uint64.valueOf(new BigInteger(segmentationId)), macAddress, port);
 213             }
 214         }
 215     }
 216
 217     public Uint64 designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<Uint64> dpns) {
 218         Uint64 designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 219         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 220             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 221                     elanInstanceName);
 222             return designatedDpnId;
 223         }
 224         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 225     }
 226
 227     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<Uint64> dpns,
 228             final Uint64 designatedDpnId, final String vmMacAddress) {
 229         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 230                 elanInstanceName, designatedDpnId, vmMacAddress);
 231
 232         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 233         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 234             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 235                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 236                 return Collections.singletonList(
 237                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 238                         dpns.remove(designatedDpnId);
 239                         for (Uint64 dpn : dpns) {
 240                             installDhcpDropAction(dpn, vmMacAddress, tx);
 241                         }
 242                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 243                     }));
 244             } else {
 245                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 246             }
 247
 248             return Collections.emptyList();
 249         });
 250
 251         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 252     }
 253
 254     public void installDhcpFlowsForVms(Uint64 designatedDpnId, Set<String> listVmMacAddress,
 255             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 256         for (String vmMacAddress : listVmMacAddress) {
 257             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 258         }
 259     }
 260
 261     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<Uint64> dpns, String vmMacAddress) {
 262         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 263         removeFromLocalCache(elanInstanceName, vmMacAddress);
 264     }
 265
 266     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<Uint64> dpns) {
 267         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 268         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 269         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 270                 elanInstanceName, tunnelIp, dpns, vmMacs);
 271         if (vmMacs == null) {
 272             return;
 273         }
 274         for (String vmMacAddress : vmMacs) {
 275             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 276         }
 277         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 278     }
 279
 280     @Nullable
 281     public Uint64 readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 282         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 283             return null;
 284         }
 285         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 286                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 287                         .child(DesignatedSwitchForTunnel.class,
 288                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 289         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional =
 290                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 291         if (designatedSwitchForTunnelOptional.isPresent()) {
 292             return Uint64.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 293         }
 294         return null;
 295     }
 296
 297     public void writeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 298                                                        String elanInstanceName) {
 299         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 300                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 301         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 302                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 303                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 304         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 305                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 306                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 307                         .withKey(designatedSwitchForTunnelKey).build();
 308         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 309                 dpnId);
 310         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 311         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 312     }
 313
 314     public void removeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 315                                                         String elanInstanceName) {
 316         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 317                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 318         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 319                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 320                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 321         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 322                 elanInstanceName, dpnId);
 323         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 324         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 325     }
 326
 327     // This method is called whenever new OVS Switch is added.
 328     public void installDhcpDropActionOnDpn(Uint64 dpId) {
 329         // During controller restart we'll get add for designatedDpns as well and we
 330         // need not install drop flows for those dpns
 331         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 332             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 333             return;
 334         }
 335         // Read from DS since the cache may not get loaded completely in restart scenario
 336         if (isDpnDesignatedDpn(dpId)) {
 337             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 338             return;
 339         }
 340         List<String> vmMacs = getAllVmMacs();
 341         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 342         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 343             for (String vmMacAddress : vmMacs) {
 344                 installDhcpDropAction(dpId, vmMacAddress, tx);
 345             }
 346         }), LOG, "Error writing to the datastore");
 347     }
 348
 349     private boolean isDpnDesignatedDpn(Uint64 dpId) {
 350         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 351                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 352         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 353                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 354         if (designatedSwitchForTunnelOptional.isPresent()) {
 355             List<DesignatedSwitchForTunnel> list =
 356                     designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 357             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 358                 if (dpId.equals(Uint64.valueOf(designatedSwitchForTunnel.getDpId()))) {
 359                     return true;
 360                 }
 361             }
 362         }
 363         return false;
 364     }
 365
 366     private List<String> getAllVmMacs() {
 367         List<String> vmMacs = new LinkedList<>();
 368         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 369         for (Set<String> list : listOfVmMacs) {
 370             vmMacs.addAll(list);
 371         }
 372         return vmMacs;
 373     }
 374
 375     public void updateLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 376         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 377         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 378         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 379         if (tunnelIpElanNameSet == null) {
 380             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 381         }
 382         tunnelIpElanNameSet.add(tunnelIpElanName);
 383         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 384                 tunnelIpElanNameSet);
 385         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 386     }
 387
 388     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 389         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 390         Set<String> setOfExistingVmMacAddress;
 391         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 392         if (setOfExistingVmMacAddress == null) {
 393             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 394         }
 395         setOfExistingVmMacAddress.add(vmMacAddress);
 396         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 397                 setOfExistingVmMacAddress);
 398         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 399         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 400     }
 401
 402     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 403         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 404         Set<String> listExistingVmMacAddress;
 405         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 406         if (listExistingVmMacAddress == null) {
 407             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 408         }
 409         listExistingVmMacAddress.add(vmMacAddress);
 410         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 411                 listExistingVmMacAddress);
 412         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 413     }
 414
 415     public void handleDesignatedDpnDown(Uint64 dpnId, List<Uint64> listOfDpns) {
 416         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 417         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 418         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 419             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 420         } else {
 421             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 422                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 423                     List<String> listOfVms = getAllVmMacs();
 424                     for (String vmMacAddress : listOfVms) {
 425                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 426                     }
 427                 }
 428                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 429                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 430                 }
 431             }), LOG, "Error writing to datastore");
 432         }
 433     }
 434
 435     public void updateCacheAndInstallNewFlows(List<Uint64> listOfDpns, Pair<IpAddress, String> pair,
 436             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 437         Uint64 newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 438         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 439             return;
 440         }
 441         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 442         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 443             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 444             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 445         }
 446         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 447         if (subnetDhcpData.isPresent()) {
 448             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 449                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 450         }
 451     }
 452
 453     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, Uint64 dpnId,
 454                                           TypedReadWriteTransaction<Configuration> tx)
 455             throws ExecutionException, InterruptedException {
 456         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 457         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 458             return;
 459         }
 460         for (String vmMacAddress : setOfVmMacAddress) {
 461             installDhcpDropAction(dpnId, vmMacAddress, tx);
 462         }
 463     }
 464
 465     /**
 466      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 467      * @param tunnelIp The tunnel Ip address
 468      * @param elanInstanceName The elan instance name
 469      * @param dpns The data path nodes
 470      * @return The designated dpn
 471      */
 472     private Uint64 chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 473             List<Uint64> dpns) {
 474         Uint64 designatedDpnId = DhcpMConstants.INVALID_DPID;
 475         if (dpns != null && dpns.size() != 0) {
 476             List<Uint64> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 477             candidateDpns.retainAll(dpns);
 478             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 479                     tunnelIp, elanInstanceName, candidateDpns);
 480             boolean elanDpnAvailableFlag = true;
 481             if (candidateDpns.isEmpty()) {
 482                 candidateDpns = dpns;
 483                 elanDpnAvailableFlag = false;
 484             }
 485             int size = 0;
 486             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 487             if (device == null) {
 488                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 489                         elanInstanceName, tunnelIp);
 490                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 491                 return designatedDpnId;
 492             }
 493             for (Uint64 dpn : candidateDpns) {
 494                 String hwvtepNodeId = device.getHwvtepNodeId();
 495                 if (!elanDpnAvailableFlag) {
 496                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 497                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 498                         continue;
 499                     }
 500                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 501                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 502                     continue;
 503                 }
 504                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 505                 if (tunnelIpElanNameSet == null) {
 506                     designatedDpnId = dpn;
 507                     break;
 508                 }
 509                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 510                     size = tunnelIpElanNameSet.size();
 511                     designatedDpnId = dpn;
 512                 }
 513             }
 514             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 515             return designatedDpnId;
 516         }
 517         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 518         return designatedDpnId;
 519     }
 520
 521     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 522         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 523     }
 524
 525     private void installDhcpEntries(Uint64 dpnId, String vmMacAddress,
 526             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 527         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 528                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 529     }
 530
 531     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 532                                               String dhcpMacAddress) {
 533         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 534         for (Entry<Uint64, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 535             Uint64 dpn = entry.getKey();
 536             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 537             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 538                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 539                     if (addFlow) {
 540                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 541                                 elanInstanceName, pair.getLeft());
 542                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 543                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 544                     } else {
 545                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 546                                 elanInstanceName, pair.getLeft());
 547                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 548                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 549                     }
 550                 }
 551             }
 552         }
 553     }
 554
 555
 556     public void configureDhcpArpRequestResponseFlow(Uint64 dpnId, String elanInstanceName, boolean addFlow,
 557                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 558         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 559         if (device == null) {
 560             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 561                     elanInstanceName);
 562         }
 563         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 564             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 565                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 566                 String tunnelInterfaceName = getExternalTunnelInterfaceName(dpnId.toString(),
 567                         device.getHwvtepNodeId());
 568                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 569                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 570                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 571                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 572                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 573                 if (optElan.isPresent()) {
 574                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 575                             lportTag, optElan.get().getElanTag());
 576                     Uuid nwUuid = new Uuid(elanInstanceName);
 577                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 578                     Uint64 vni = strVni != null ? Uint64.valueOf(strVni) : Uint64.valueOf(0);
 579                     if (!vni.equals(Uint64.ZERO)) {
 580                         return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 581                             Datastore.CONFIGURATION, tx -> {
 582                                 if (addFlow) {
 583                                     LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 584                                         dpnId, dhcpIpAddress, lportTag);
 585                                     installDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag,
 586                                         optElan.get().getElanTag().toJava());
 587                                     installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 588                                         dhcpIpAddress, dhcpMacAddress);
 589                                 } else {
 590                                     LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 591                                         dpnId, dhcpIpAddress, lportTag);
 592                                     uninstallDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag);
 593                                     uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 594                                 }
 595                             }));
 596                     }
 597                 }
 598             }
 599             return Collections.emptyList();
 600         });
 601     }
 602
 603     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 604         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 605         Uuid nwUuid = new Uuid(elanInstanceName);
 606         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 607         for (Uuid subnet : subnets) {
 608             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 609                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 610                 return optSubnetDhcp;
 611             }
 612         }
 613         return optSubnetDhcp;
 614     }
 615
 616     private void installDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 617                                             Uint64 vni, String dhcpIpAddress, int lportTag, Long elanTag)
 618             throws ExecutionException, InterruptedException {
 619         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 620                 lportTag, elanTag, true, mdsalUtil, tx);
 621     }
 622
 623     private void installDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 624                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 625         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 626                 elanInstanceName, lportTag, dhcpIpAddress);
 627         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 628         builder.setDpId(dpnId.toJava()).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 629                 .setLportTag(lportTag);
 630         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 631                 dhcpIpAddress, dhcpMacAddress, itmRpcService));
 632         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 633     }
 634
 635     private void uninstallDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 636                                                 String dhcpIpAddress) {
 637         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 638         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId.toJava())
 639                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 640                 .setLportTag(lportTag).buildForRemoveFlow();
 641         elanService.removeArpResponderFlow(arpInput);
 642     }
 643
 644     private void uninstallDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 645                                               Uint64 vni, String dhcpIpAddress, int lportTag)
 646             throws ExecutionException, InterruptedException {
 647         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 648                 lportTag, null, false, mdsalUtil, tx);
 649     }
 650
 651
 652     public void unInstallDhcpEntries(Uint64 dpnId, String vmMacAddress,
 653             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 654         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 655                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 656     }
 657
 658     private void installDhcpDropAction(Uint64 dpn, String vmMacAddress,
 659             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 660         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 661                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 662     }
 663
 664     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, Uint64 interfaceDpn) {
 665         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 666         if (interfaceDpn == null) {
 667             return Collections.emptyList();
 668         }
 669         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 670             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 671                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 672             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 673                 return Collections.emptyList();
 674             }
 675             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 676                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 677                 if (tunnelIpInDpn.equals(tunnelIp)) {
 678                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 679                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 680                                 tunnelElanPair);
 681                         return Collections.emptyList();
 682                     }
 683                 }
 684             }
 685             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 686                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 687                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 688                     String elanInstanceName = tunnelElanPair.getRight();
 689                     if (tunnelIpInDpn.equals(tunnelIp)) {
 690                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 691                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 692                                     tunnelElanPair);
 693                         }
 694                         List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 695                         dpns.remove(interfaceDpn);
 696                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 697                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 698                         if (subnetDhcpData.isPresent()) {
 699                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 700                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 701                                     subnetDhcpData.get().getPortMacaddress());
 702                         }
 703                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 704                     }
 705                 }
 706             }));
 707         }
 708     }
 709
 710     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 711         for (L2GatewayDevice device : ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight())) {
 712             if (Objects.equals(device.getTunnelIp(), tunnelElanPair.getLeft())) {
 713                 return true;
 714             }
 715         }
 716         return false;
 717     }
 718
 719     private String getTunnelIpDpnKey(IpAddress tunnelIp, Uint64 interfaceDpn) {
 720         return tunnelIp.toString() + interfaceDpn;
 721     }
 722
 723     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 724         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 725             Pair<IpAddress, String> pair = entry.getKey();
 726             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 727                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 728                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 729                     continue;
 730                 }
 731                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 732                         setOfExistingVmMacAddress, elanInstanceName);
 733                 setOfExistingVmMacAddress.remove(vmMacAddress);
 734                 if (setOfExistingVmMacAddress.size() > 0) {
 735                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 736                     return;
 737                 }
 738                 tunnelIpElanNameToVmMacCache.remove(pair);
 739             }
 740         }
 741     }
 742
 743     public void removeFromLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 744         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 745         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 746         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 747         if (tunnelIpElanNameSet != null) {
 748             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 749                             + "designatedDpnId {}",
 750                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 751             tunnelIpElanNameSet.remove(tunnelIpElanName);
 752             if (tunnelIpElanNameSet.size() != 0) {
 753                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 754             } else {
 755                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 756             }
 757         }
 758     }
 759
 760     public void updateVniMacToPortCache(Uint64 vni, String macAddress, Port port) {
 761         if (macAddress == null) {
 762             return;
 763         }
 764         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 765                 vni, macAddress.toUpperCase(Locale.getDefault()));
 766         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 767                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 768         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 769     }
 770
 771     public void removeVniMacToPortCache(Uint64 vni, String macAddress) {
 772         if (macAddress == null) {
 773             return;
 774         }
 775         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 776                 vni, macAddress.toUpperCase(Locale.getDefault()));
 777         vniMacAddressToPortCache.remove(vniMacAddressPair);
 778     }
 779
 780     @Nullable
 781     public Port readVniMacToPortCache(Uint64 vni, String macAddress) {
 782         if (macAddress == null) {
 783             return null;
 784         }
 785         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 786                 vni, macAddress.toUpperCase(Locale.getDefault()));
 787         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 788                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 789                 vniMacAddressToPortCache.get(vniMacAddressPair));
 790         return vniMacAddressToPortCache.get(vniMacAddressPair);
 791     }
 792
 793     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 794         String tunnelInterfaceName = null;
 795         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 796         try {
 797             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 798                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 799                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 800
 801             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 802             if (rpcResult.isSuccessful()) {
 803                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 804                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 805             } else {
 806                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 807             }
 808         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 809             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 810                     sourceNode, dstNode, e);
 811         }
 812         return tunnelInterfaceName;
 813     }
 814
 815     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 816         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 817                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 818         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 819     }
 820
 821     @Nullable
 822     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 823         Set<LocatorSet> locators = new HashSet<>();
 824         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 825                 internalTunnelIp.getIpv4Address().getValue());
 826         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 827                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 828                         terminationPointKey));
 829         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 830
 831         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 832                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 833
 834         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 835                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 836                 .setLogicalSwitchRef(lsRef).build();
 837         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 838                 dstDevice.getNodeId(), remoteMcastMacs.key());
 839         ReadOnlyTransaction transaction = broker.newReadOnlyTransaction();
 840         try {
 841             //TODO do async mdsal read
 842             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).checkedGet().get();
 843             locators.addAll(remoteMcastMacs.getLocatorSet());
 844             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 845         } catch (ReadFailedException e) {
 846             LOG.error("Failed to read the macs {}", iid);
 847         } finally {
 848             transaction.close();
 849         }
 850         return null;
 851     }
 852
 853     private void putRemoteMcastMac(TypedWriteTransaction<Configuration> transaction, String elanName,
 854                                    L2GatewayDevice device, IpAddress internalTunnelIp) {
 855         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 856         if (!optionalNode.isPresent()) {
 857             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 858             return;
 859         }
 860         Node dstNode = optionalNode.get();
 861         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 862         HwvtepUtils.addRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 863     }
 864
 865     public void installRemoteMcastMac(final Uint64 designatedDpnId, final IpAddress tunnelIp,
 866                                       final String elanInstanceName) {
 867         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 868             return;
 869         }
 870
 871         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 872             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 873                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 874                 LOG.info("Installing remote McastMac is not executed for this node.");
 875                 return Collections.emptyList();
 876             }
 877
 878             LOG.info("Installing remote McastMac");
 879             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 880             if (device == null) {
 881                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 882                     elanInstanceName);
 883                 return Collections.emptyList();
 884             }
 885             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 886                     device.getHwvtepNodeId());
 887             if (tunnelInterfaceName != null) {
 888                 Interface tunnelInterface =
 889                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 890                 if (tunnelInterface == null) {
 891                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 892                     return Collections.emptyList();
 893                 }
 894                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 895                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 896                             tunnelInterface.augmentation(IfTunnel.class).getTunnelSource())));
 897             }
 898             return Collections.emptyList();
 899         });
 900     }
 901
 902     @Nullable
 903     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 904         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 905         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 906         for (L2GatewayDevice device : devices) {
 907             if (tunnelIp.equals(device.getTunnelIp())) {
 908                 return device;
 909             }
 910         }
 911         return null;
 912     }
 913
 914     private boolean isTunnelUp(String nodeName, Uint64 dpn) {
 915         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 916         if (tunnelInterfaceName == null) {
 917             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 918             return false;
 919         }
 920         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 921                 .Interface tunnelInterface =
 922                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 923         if (tunnelInterface == null) {
 924             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 925             return false;
 926         }
 927         return tunnelInterface.getOperStatus() == OperStatus.Up;
 928     }
 929
 930     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, Uint64 interfaceDpn) {
 931         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 932         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 933             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 934                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 935             List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 936             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 937                 LOG.trace("There are no undesignated DPNs");
 938                 return Collections.emptyList();
 939             }
 940             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 941                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 942                     if (tunnelIp.equals(pair.getLeft())) {
 943                         String elanInstanceName = pair.getRight();
 944                         Uint64 newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 945                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 946                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 947                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 948                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 949                                         vmMacAddress, newDesignatedDpn);
 950                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 951                             }
 952                         }
 953                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 954                         if (subnetDhcpData.isPresent()) {
 955                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 956                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 957                                     subnetDhcpData.get().getPortMacaddress());
 958                         }
 959                     }
 960                 }
 961             }));
 962         }
 963     }
 964
 965     private boolean isTunnelConfigured(Uint64 dpn, String hwVtepNodeId) {
 966         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 967         if (tunnelInterfaceName == null) {
 968             return false;
 969         }
 970         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 971         if (tunnelInterface == null) {
 972             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 973             return false;
 974         }
 975         return true;
 976     }
 977
 978     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 979         availableVMCache.remove(tunnelIpElanName);
 980     }
 981
 982     private void unInstallDhcpEntriesOnDpns(final List<Uint64> dpns, final String vmMacAddress) {
 983         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
 984             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 985                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 986                 return Collections.singletonList(
 987                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 988                         for (final Uint64 dpn : dpns) {
 989                             unInstallDhcpEntries(dpn, vmMacAddress, tx);
 990                         }
 991                     }));
 992             } else {
 993                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 994             }
 995
 996             return Collections.emptyList();
 997         });
 998     }
 999
1000     @Nullable
1001     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
1002         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
1003         IpAddress tunnelIp = null;
1004         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
1005             Pair<IpAddress, String> pair = entry.getKey();
1006             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
1007                     pair.getRight());
1008             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
1009                 Set<String> listExistingVmMacAddress = entry.getValue();
1010                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
1011                         && listExistingVmMacAddress.contains(vmMacAddress)) {
1012                     tunnelIp = pair.getLeft();
1013                     break;
1014                 }
1015             }
1016         }
1017         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
1018         return tunnelIp;
1019     }
1020
1021     private String getJobKey(final String jobKeySuffix) {
1022         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
1023     }
1024 }