dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.util.concurrent.ListenableFuture;
  14 import java.math.BigInteger;
  15 import java.util.ArrayList;
  16 import java.util.Collection;
  17 import java.util.Collections;
  18 import java.util.HashSet;
  19 import java.util.LinkedList;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map.Entry;
  23 import java.util.Objects;
  24 import java.util.Set;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentMap;
  27 import java.util.concurrent.CopyOnWriteArraySet;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import javax.annotation.PostConstruct;
  31 import javax.inject.Inject;
  32 import javax.inject.Named;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.tuple.ImmutablePair;
  35 import org.apache.commons.lang3.tuple.Pair;
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.infra.Datastore;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  46 import org.opendaylight.genius.infra.TypedWriteTransaction;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.NwConstants;
  50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  51 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  52 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  53 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  54 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  57 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  58 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  59 import org.opendaylight.netvirt.dhcpservice.api.IDhcpExternalTunnelManager;
  60 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  61 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  64 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  65 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  66 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  97 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  98 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  99 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
 100 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 101 import org.opendaylight.yangtools.yang.common.RpcResult;
 102 import org.slf4j.Logger;
 103 import org.slf4j.LoggerFactory;
 104
 105 @Singleton
 106 public class DhcpExternalTunnelManager implements IDhcpExternalTunnelManager {
 107
 108     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 109     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 110
 111     private final DataBroker broker;
 112     private final ManagedNewTransactionRunner txRunner;
 113     private final IMdsalApiManager mdsalUtil;
 114     private final ItmRpcService itmRpcService;
 115     private final EntityOwnershipUtils entityOwnershipUtils;
 116     private final IInterfaceManager interfaceManager;
 117     private final JobCoordinator jobCoordinator;
 118     private final L2GatewayCache l2GatewayCache;
 119     private IElanService elanService;
 120     private final DhcpServiceCounters dhcpServiceCounters;
 121
 122     private final ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 123             new ConcurrentHashMap<>();
 124     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 125             new ConcurrentHashMap<>();
 126     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 127     private final ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 128
 129     @Override
 130     public ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> getDesignatedDpnsToTunnelIpElanNameCache() {
 131         return designatedDpnsToTunnelIpElanNameCache;
 132     }
 133
 134     @Override
 135     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getTunnelIpElanNameToVmMacCache() {
 136         return tunnelIpElanNameToVmMacCache;
 137     }
 138
 139     @Override
 140     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getAvailableVMCache() {
 141         return availableVMCache;
 142     }
 143
 144     @Override
 145     public ConcurrentMap<Pair<BigInteger, String>, Port> getVniMacAddressToPortCache() {
 146         return vniMacAddressToPortCache;
 147     }
 148
 149     @Inject
 150     public DhcpExternalTunnelManager(final DataBroker broker,
 151             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 152             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 153             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 154             @Named("elanService") IElanService ielanService, DhcpServiceCounters dhcpServiceCounters) {
 155         this.broker = broker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 157         this.mdsalUtil = mdsalUtil;
 158         this.itmRpcService = itmRpcService;
 159         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 160         this.interfaceManager = interfaceManager;
 161         this.jobCoordinator = jobCoordinator;
 162         this.l2GatewayCache = l2GatewayCache;
 163         this.elanService = ielanService;
 164         this.dhcpServiceCounters = dhcpServiceCounters;
 165     }
 166
 167     @PostConstruct
 168     public void init() {
 169         initilizeCaches();
 170     }
 171
 172     private void initilizeCaches() {
 173         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 174         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 175                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 176         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 177                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 178         if (designatedSwitchForTunnelOptional.isPresent()) {
 179             List<DesignatedSwitchForTunnel> list =
 180                 designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 181             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 182                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 183                         designatedDpnsToTunnelIpElanNameCache
 184                                 .get(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()));
 185                 if (setOfTunnelIpElanNamePair == null) {
 186                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 187                 }
 188                 Pair<IpAddress, String> tunnelIpElanNamePair =
 189                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 190                                 designatedSwitchForTunnel.getElanInstanceName());
 191                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 192                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()),
 193                         setOfTunnelIpElanNamePair);
 194             }
 195         }
 196         LOG.trace("Loading vniMacAddressToPortCache");
 197         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 198         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 199         if (optionalPorts.isPresent()) {
 200             List<Port> list = optionalPorts.get().nonnullPort();
 201             for (Port port : list) {
 202                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 203                     continue;
 204                 }
 205                 String macAddress = port.getMacAddress().getValue();
 206                 Uuid networkId = port.getNetworkId();
 207                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 208                 if (segmentationId == null) {
 209                     return;
 210                 }
 211                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 212             }
 213         }
 214     }
 215
 216     public BigInteger designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns) {
 217         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 218         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 219             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 220                     elanInstanceName);
 221             return designatedDpnId;
 222         }
 223         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 224     }
 225
 226     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<BigInteger> dpns,
 227             final BigInteger designatedDpnId, final String vmMacAddress) {
 228         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 229                 elanInstanceName, designatedDpnId, vmMacAddress);
 230
 231         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 232         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 233             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 234                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 235                 return Collections.singletonList(
 236                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 237                         dpns.remove(designatedDpnId);
 238                         for (BigInteger dpn : dpns) {
 239                             installDhcpDropAction(dpn, vmMacAddress, tx);
 240                         }
 241                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 242                     }));
 243             } else {
 244                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 245             }
 246
 247             return Collections.emptyList();
 248         });
 249
 250         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 251     }
 252
 253     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress,
 254             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 255         for (String vmMacAddress : listVmMacAddress) {
 256             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 257         }
 258     }
 259
 260     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 261         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 262         removeFromLocalCache(elanInstanceName, vmMacAddress);
 263     }
 264
 265     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 266         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 267         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 268         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 269                 elanInstanceName, tunnelIp, dpns, vmMacs);
 270         if (vmMacs == null) {
 271             return;
 272         }
 273         for (String vmMacAddress : vmMacs) {
 274             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 275         }
 276         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 277     }
 278
 279     @Nullable
 280     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 281         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 282             return null;
 283         }
 284         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 285                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 286                         .child(DesignatedSwitchForTunnel.class,
 287                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 288         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional =
 289                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 290         if (designatedSwitchForTunnelOptional.isPresent()) {
 291             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 292         }
 293         return null;
 294     }
 295
 296     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 297                                                        String elanInstanceName) {
 298         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 299                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 300         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 301                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 302                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 303         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 304                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 305                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 306                         .withKey(designatedSwitchForTunnelKey).build();
 307         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 308                 dpnId);
 309         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 310         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 311     }
 312
 313     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 314                                                         String elanInstanceName) {
 315         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 316                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 317         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 318                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 319                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 320         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 321                 elanInstanceName, dpnId);
 322         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 323         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 324     }
 325
 326     // This method is called whenever new OVS Switch is added.
 327     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 328         // During controller restart we'll get add for designatedDpns as well and we
 329         // need not install drop flows for those dpns
 330         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 331             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 332             return;
 333         }
 334         // Read from DS since the cache may not get loaded completely in restart scenario
 335         if (isDpnDesignatedDpn(dpId)) {
 336             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 337             return;
 338         }
 339         List<String> vmMacs = getAllVmMacs();
 340         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 341         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 342             for (String vmMacAddress : vmMacs) {
 343                 installDhcpDropAction(dpId, vmMacAddress, tx);
 344             }
 345         }), LOG, "Error writing to the datastore");
 346     }
 347
 348     private boolean isDpnDesignatedDpn(BigInteger dpId) {
 349         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 350                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 351         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 352                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 353         if (designatedSwitchForTunnelOptional.isPresent()) {
 354             List<DesignatedSwitchForTunnel> list =
 355                     designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 356             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 357                 if (dpId.equals(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()))) {
 358                     return true;
 359                 }
 360             }
 361         }
 362         return false;
 363     }
 364
 365     private List<String> getAllVmMacs() {
 366         List<String> vmMacs = new LinkedList<>();
 367         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 368         for (Set<String> list : listOfVmMacs) {
 369             vmMacs.addAll(list);
 370         }
 371         return vmMacs;
 372     }
 373
 374     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 375         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 376         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 377         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 378         if (tunnelIpElanNameSet == null) {
 379             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 380         }
 381         tunnelIpElanNameSet.add(tunnelIpElanName);
 382         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 383                 tunnelIpElanNameSet);
 384         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 385     }
 386
 387     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 388         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 389         Set<String> setOfExistingVmMacAddress;
 390         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 391         if (setOfExistingVmMacAddress == null) {
 392             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 393         }
 394         setOfExistingVmMacAddress.add(vmMacAddress);
 395         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 396                 setOfExistingVmMacAddress);
 397         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 398         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 399     }
 400
 401     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 402         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 403         Set<String> listExistingVmMacAddress;
 404         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 405         if (listExistingVmMacAddress == null) {
 406             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 407         }
 408         listExistingVmMacAddress.add(vmMacAddress);
 409         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 410                 listExistingVmMacAddress);
 411         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 412     }
 413
 414     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 415         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 416         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 417         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 418             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 419         } else {
 420             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 421                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 422                     List<String> listOfVms = getAllVmMacs();
 423                     for (String vmMacAddress : listOfVms) {
 424                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 425                     }
 426                 }
 427                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 428                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 429                 }
 430             }), LOG, "Error writing to datastore");
 431         }
 432     }
 433
 434     public void updateCacheAndInstallNewFlows(List<BigInteger> listOfDpns, Pair<IpAddress, String> pair,
 435             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 436         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 437         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 438             return;
 439         }
 440         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 441         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 442             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 443             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 444         }
 445         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 446         if (subnetDhcpData.isPresent()) {
 447             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 448                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 449         }
 450     }
 451
 452     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId,
 453                                           TypedReadWriteTransaction<Configuration> tx)
 454             throws ExecutionException, InterruptedException {
 455         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 456         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 457             return;
 458         }
 459         for (String vmMacAddress : setOfVmMacAddress) {
 460             installDhcpDropAction(dpnId, vmMacAddress, tx);
 461         }
 462     }
 463
 464     /**
 465      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 466      * @param tunnelIp The tunnel Ip address
 467      * @param elanInstanceName The elan instance name
 468      * @param dpns The data path nodes
 469      * @return The designated dpn
 470      */
 471     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 472             List<BigInteger> dpns) {
 473         BigInteger designatedDpnId = DhcpMConstants.INVALID_DPID;
 474         if (dpns != null && dpns.size() != 0) {
 475             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 476             candidateDpns.retainAll(dpns);
 477             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 478                     tunnelIp, elanInstanceName, candidateDpns);
 479             boolean elanDpnAvailableFlag = true;
 480             if (candidateDpns.isEmpty()) {
 481                 candidateDpns = dpns;
 482                 elanDpnAvailableFlag = false;
 483             }
 484             int size = 0;
 485             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 486             if (device == null) {
 487                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 488                         elanInstanceName, tunnelIp);
 489                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 490                 return designatedDpnId;
 491             }
 492             for (BigInteger dpn : candidateDpns) {
 493                 String hwvtepNodeId = device.getHwvtepNodeId();
 494                 if (!elanDpnAvailableFlag) {
 495                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 496                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 497                         continue;
 498                     }
 499                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 500                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 501                     continue;
 502                 }
 503                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 504                 if (tunnelIpElanNameSet == null) {
 505                     designatedDpnId = dpn;
 506                     break;
 507                 }
 508                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 509                     size = tunnelIpElanNameSet.size();
 510                     designatedDpnId = dpn;
 511                 }
 512             }
 513             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 514             return designatedDpnId;
 515         }
 516         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 517         return designatedDpnId;
 518     }
 519
 520     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 521         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 522     }
 523
 524     private void installDhcpEntries(BigInteger dpnId, String vmMacAddress,
 525             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 526         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 527                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 528     }
 529
 530     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 531                                               String dhcpMacAddress) {
 532         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 533         for (Entry<BigInteger, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 534             BigInteger dpn = entry.getKey();
 535             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 536             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 537                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 538                     if (addFlow) {
 539                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 540                                 elanInstanceName, pair.getLeft());
 541                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 542                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 543                     } else {
 544                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 545                                 elanInstanceName, pair.getLeft());
 546                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 547                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 548                     }
 549                 }
 550             }
 551         }
 552     }
 553
 554
 555     public void configureDhcpArpRequestResponseFlow(BigInteger dpnId, String elanInstanceName, boolean addFlow,
 556                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 557         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 558         if (device == null) {
 559             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 560                     elanInstanceName);
 561         }
 562         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 563             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 564                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 565                 String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpnId),
 566                         device.getHwvtepNodeId());
 567                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 568                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 569                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 570                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 571                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 572                 if (optElan.isPresent()) {
 573                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 574                             lportTag, optElan.get().getElanTag());
 575                     Uuid nwUuid = new Uuid(elanInstanceName);
 576                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 577                     BigInteger vni = strVni != null ? new BigInteger(strVni) : BigInteger.ZERO;
 578                     if (!vni.equals(BigInteger.ZERO)) {
 579                         return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 580                             Datastore.CONFIGURATION, tx -> {
 581                                 if (addFlow) {
 582                                     LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 583                                         dpnId, dhcpIpAddress, lportTag);
 584                                     installDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag,
 585                                         optElan.get().getElanTag());
 586                                     installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 587                                         dhcpIpAddress, dhcpMacAddress);
 588                                 } else {
 589                                     LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 590                                         dpnId, dhcpIpAddress, lportTag);
 591                                     uninstallDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag);
 592                                     uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 593                                 }
 594                             }));
 595                     }
 596                 }
 597             }
 598             return Collections.emptyList();
 599         });
 600     }
 601
 602     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 603         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 604         Uuid nwUuid = new Uuid(elanInstanceName);
 605         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 606         for (Uuid subnet : subnets) {
 607             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 608                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 609                 return optSubnetDhcp;
 610             }
 611         }
 612         return optSubnetDhcp;
 613     }
 614
 615     private void installDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, BigInteger dpnId,
 616                                             BigInteger vni, String dhcpIpAddress, int lportTag, Long elanTag)
 617             throws ExecutionException, InterruptedException {
 618         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 619                 lportTag, elanTag, true, mdsalUtil, tx);
 620     }
 621
 622     private void installDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 623                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 624         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 625                 elanInstanceName, lportTag, dhcpIpAddress);
 626         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 627         builder.setDpId(dpnId).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 628                 .setLportTag(lportTag);
 629         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 630                 dhcpIpAddress, dhcpMacAddress, itmRpcService));
 631         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 632     }
 633
 634     private void uninstallDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 635                                                 String dhcpIpAddress) {
 636         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 637         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId)
 638                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 639                 .setLportTag(lportTag).buildForRemoveFlow();
 640         elanService.removeArpResponderFlow(arpInput);
 641     }
 642
 643     private void uninstallDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, BigInteger dpnId,
 644                                               BigInteger vni, String dhcpIpAddress, int lportTag)
 645             throws ExecutionException, InterruptedException {
 646         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 647                 lportTag, null, false, mdsalUtil, tx);
 648     }
 649
 650
 651     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress,
 652             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 653         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 654                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 655     }
 656
 657     private void installDhcpDropAction(BigInteger dpn, String vmMacAddress,
 658             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 659         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 660                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 661     }
 662
 663     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn) {
 664         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 665         if (interfaceDpn == null) {
 666             return Collections.emptyList();
 667         }
 668         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 669             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 670                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 671             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 672                 return Collections.emptyList();
 673             }
 674             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 675                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 676                 if (tunnelIpInDpn.equals(tunnelIp)) {
 677                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 678                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 679                                 tunnelElanPair);
 680                         return Collections.emptyList();
 681                     }
 682                 }
 683             }
 684             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 685                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 686                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 687                     String elanInstanceName = tunnelElanPair.getRight();
 688                     if (tunnelIpInDpn.equals(tunnelIp)) {
 689                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 690                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 691                                     tunnelElanPair);
 692                         }
 693                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 694                         dpns.remove(interfaceDpn);
 695                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 696                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 697                         if (subnetDhcpData.isPresent()) {
 698                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 699                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 700                                     subnetDhcpData.get().getPortMacaddress());
 701                         }
 702                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 703                     }
 704                 }
 705             }));
 706         }
 707     }
 708
 709     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 710         for (L2GatewayDevice device : ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight())) {
 711             if (Objects.equals(device.getTunnelIp(), tunnelElanPair.getLeft())) {
 712                 return true;
 713             }
 714         }
 715         return false;
 716     }
 717
 718     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 719         return tunnelIp.toString() + interfaceDpn;
 720     }
 721
 722     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 723         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 724             Pair<IpAddress, String> pair = entry.getKey();
 725             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 726                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 727                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 728                     continue;
 729                 }
 730                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 731                         setOfExistingVmMacAddress, elanInstanceName);
 732                 setOfExistingVmMacAddress.remove(vmMacAddress);
 733                 if (setOfExistingVmMacAddress.size() > 0) {
 734                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 735                     return;
 736                 }
 737                 tunnelIpElanNameToVmMacCache.remove(pair);
 738             }
 739         }
 740     }
 741
 742     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 743         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 744         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 745         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 746         if (tunnelIpElanNameSet != null) {
 747             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 748                             + "designatedDpnId {}",
 749                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 750             tunnelIpElanNameSet.remove(tunnelIpElanName);
 751             if (tunnelIpElanNameSet.size() != 0) {
 752                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 753             } else {
 754                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 755             }
 756         }
 757     }
 758
 759     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 760         if (macAddress == null) {
 761             return;
 762         }
 763         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 764                 vni, macAddress.toUpperCase(Locale.getDefault()));
 765         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 766                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 767         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 768     }
 769
 770     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 771         if (macAddress == null) {
 772             return;
 773         }
 774         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 775                 vni, macAddress.toUpperCase(Locale.getDefault()));
 776         vniMacAddressToPortCache.remove(vniMacAddressPair);
 777     }
 778
 779     @Nullable
 780     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 781         if (macAddress == null) {
 782             return null;
 783         }
 784         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 785                 vni, macAddress.toUpperCase(Locale.getDefault()));
 786         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 787                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 788                 vniMacAddressToPortCache.get(vniMacAddressPair));
 789         return vniMacAddressToPortCache.get(vniMacAddressPair);
 790     }
 791
 792     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 793         String tunnelInterfaceName = null;
 794         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 795         try {
 796             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 797                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 798                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 799
 800             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 801             if (rpcResult.isSuccessful()) {
 802                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 803                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 804             } else {
 805                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 806             }
 807         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 808             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 809                     sourceNode, dstNode, e);
 810         }
 811         return tunnelInterfaceName;
 812     }
 813
 814     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 815         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 816                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 817         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 818     }
 819
 820     @Nullable
 821     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 822         Set<LocatorSet> locators = new HashSet<>();
 823         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 824                 internalTunnelIp.getIpv4Address().getValue());
 825         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 826                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 827                         terminationPointKey));
 828         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 829
 830         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 831                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 832
 833         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 834                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 835                 .setLogicalSwitchRef(lsRef).build();
 836         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 837                 dstDevice.getNodeId(), remoteMcastMacs.key());
 838         ReadOnlyTransaction transaction = broker.newReadOnlyTransaction();
 839         try {
 840             //TODO do async mdsal read
 841             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).checkedGet().get();
 842             locators.addAll(remoteMcastMacs.getLocatorSet());
 843             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 844         } catch (ReadFailedException e) {
 845             LOG.error("Failed to read the macs {}", iid);
 846         }
 847         return null;
 848     }
 849
 850     private void putRemoteMcastMac(TypedWriteTransaction<Configuration> transaction, String elanName,
 851                                    L2GatewayDevice device, IpAddress internalTunnelIp) {
 852         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 853         if (!optionalNode.isPresent()) {
 854             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 855             return;
 856         }
 857         Node dstNode = optionalNode.get();
 858         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 859         HwvtepUtils.addRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 860     }
 861
 862     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp,
 863                                       final String elanInstanceName) {
 864         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 865             return;
 866         }
 867
 868         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 869             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 870                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 871                 LOG.info("Installing remote McastMac is not executed for this node.");
 872                 return Collections.emptyList();
 873             }
 874
 875             LOG.info("Installing remote McastMac");
 876             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 877             if (device == null) {
 878                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 879                     elanInstanceName);
 880                 return Collections.emptyList();
 881             }
 882             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 883                     device.getHwvtepNodeId());
 884             if (tunnelInterfaceName != null) {
 885                 Interface tunnelInterface =
 886                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 887                 if (tunnelInterface == null) {
 888                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 889                     return Collections.emptyList();
 890                 }
 891                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 892                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 893                             tunnelInterface.augmentation(IfTunnel.class).getTunnelSource())));
 894             }
 895             return Collections.emptyList();
 896         });
 897     }
 898
 899     @Nullable
 900     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 901         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 902         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 903         for (L2GatewayDevice device : devices) {
 904             if (tunnelIp.equals(device.getTunnelIp())) {
 905                 return device;
 906             }
 907         }
 908         return null;
 909     }
 910
 911     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 912         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 913         if (tunnelInterfaceName == null) {
 914             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 915             return false;
 916         }
 917         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 918                 .Interface tunnelInterface =
 919                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 920         if (tunnelInterface == null) {
 921             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 922             return false;
 923         }
 924         return tunnelInterface.getOperStatus() == OperStatus.Up;
 925     }
 926
 927     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn) {
 928         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 929         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 930             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 931                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 932             List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 933             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 934                 LOG.trace("There are no undesignated DPNs");
 935                 return Collections.emptyList();
 936             }
 937             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 938                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 939                     if (tunnelIp.equals(pair.getLeft())) {
 940                         String elanInstanceName = pair.getRight();
 941                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 942                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 943                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 944                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 945                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 946                                         vmMacAddress, newDesignatedDpn);
 947                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 948                             }
 949                         }
 950                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 951                         if (subnetDhcpData.isPresent()) {
 952                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 953                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 954                                     subnetDhcpData.get().getPortMacaddress());
 955                         }
 956                     }
 957                 }
 958             }));
 959         }
 960     }
 961
 962     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 963         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 964         if (tunnelInterfaceName == null) {
 965             return false;
 966         }
 967         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 968         if (tunnelInterface == null) {
 969             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 970             return false;
 971         }
 972         return true;
 973     }
 974
 975     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 976         availableVMCache.remove(tunnelIpElanName);
 977     }
 978
 979     private void unInstallDhcpEntriesOnDpns(final List<BigInteger> dpns, final String vmMacAddress) {
 980         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
 981             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 982                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 983                 return Collections.singletonList(
 984                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 985                         for (final BigInteger dpn : dpns) {
 986                             unInstallDhcpEntries(dpn, vmMacAddress, tx);
 987                         }
 988                     }));
 989             } else {
 990                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 991             }
 992
 993             return Collections.emptyList();
 994         });
 995     }
 996
 997     @Nullable
 998     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 999         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
1000         IpAddress tunnelIp = null;
1001         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
1002             Pair<IpAddress, String> pair = entry.getKey();
1003             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
1004                     pair.getRight());
1005             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
1006                 Set<String> listExistingVmMacAddress = entry.getValue();
1007                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
1008                         && listExistingVmMacAddress.contains(vmMacAddress)) {
1009                     tunnelIp = pair.getLeft();
1010                     break;
1011                 }
1012             }
1013         }
1014         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
1015         return tunnelIp;
1016     }
1017
1018     private String getJobKey(final String jobKeySuffix) {
1019         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
1020     }
1021 }