dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.util.concurrent.ListenableFuture;
  13 import java.math.BigInteger;
  14 import java.util.ArrayList;
  15 import java.util.Collection;
  16 import java.util.Collections;
  17 import java.util.HashSet;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Locale;
  21 import java.util.Map.Entry;
  22 import java.util.Objects;
  23 import java.util.Optional;
  24 import java.util.Set;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentMap;
  27 import java.util.concurrent.CopyOnWriteArraySet;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import javax.annotation.PostConstruct;
  31 import javax.inject.Inject;
  32 import javax.inject.Named;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.tuple.ImmutablePair;
  35 import org.apache.commons.lang3.tuple.Pair;
  36 import org.eclipse.jdt.annotation.NonNull;
  37 import org.eclipse.jdt.annotation.Nullable;
  38 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  39 import org.opendaylight.genius.infra.Datastore;
  40 import org.opendaylight.genius.infra.Datastore.Configuration;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  44 import org.opendaylight.genius.infra.TypedWriteTransaction;
  45 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  49 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  50 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  51 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  52 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  53 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  54 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  55 import org.opendaylight.mdsal.binding.api.DataBroker;
  56 import org.opendaylight.mdsal.binding.api.ReadTransaction;
  57 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  58 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  59 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  60 import org.opendaylight.netvirt.dhcpservice.api.IDhcpExternalTunnelManager;
  61 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  62 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  63 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  64 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  65 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  66 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  67 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  98 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  99 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
 100 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
 101 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 102 import org.opendaylight.yangtools.yang.common.RpcResult;
 103 import org.opendaylight.yangtools.yang.common.Uint64;
 104 import org.slf4j.Logger;
 105 import org.slf4j.LoggerFactory;
 106
 107 @Singleton
 108 public class DhcpExternalTunnelManager implements IDhcpExternalTunnelManager {
 109
 110     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 111     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 112
 113     private final DataBroker broker;
 114     private final ManagedNewTransactionRunner txRunner;
 115     private final IMdsalApiManager mdsalUtil;
 116     private final ItmRpcService itmRpcService;
 117     private final EntityOwnershipUtils entityOwnershipUtils;
 118     private final IInterfaceManager interfaceManager;
 119     private final JobCoordinator jobCoordinator;
 120     private final L2GatewayCache l2GatewayCache;
 121     private IElanService elanService;
 122     private final DhcpServiceCounters dhcpServiceCounters;
 123
 124     private final ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 125             new ConcurrentHashMap<>();
 126     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 127             new ConcurrentHashMap<>();
 128     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 129     private final ConcurrentMap<Pair<Uint64, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 130
 131     @Override
 132     public ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> getDesignatedDpnsToTunnelIpElanNameCache() {
 133         return designatedDpnsToTunnelIpElanNameCache;
 134     }
 135
 136     @Override
 137     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getTunnelIpElanNameToVmMacCache() {
 138         return tunnelIpElanNameToVmMacCache;
 139     }
 140
 141     @Override
 142     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getAvailableVMCache() {
 143         return availableVMCache;
 144     }
 145
 146     @Override
 147     public ConcurrentMap<Pair<Uint64, String>, Port> getVniMacAddressToPortCache() {
 148         return vniMacAddressToPortCache;
 149     }
 150
 151     @Inject
 152     public DhcpExternalTunnelManager(final DataBroker broker,
 153             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 154             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 155             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 156             @Named("elanService") IElanService ielanService, DhcpServiceCounters dhcpServiceCounters) {
 157         this.broker = broker;
 158         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 159         this.mdsalUtil = mdsalUtil;
 160         this.itmRpcService = itmRpcService;
 161         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 162         this.interfaceManager = interfaceManager;
 163         this.jobCoordinator = jobCoordinator;
 164         this.l2GatewayCache = l2GatewayCache;
 165         this.elanService = ielanService;
 166         this.dhcpServiceCounters = dhcpServiceCounters;
 167     }
 168
 169     @PostConstruct
 170     public void init() {
 171         initilizeCaches();
 172     }
 173
 174     private void initilizeCaches() {
 175         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 176         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 177                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 178         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional;
 179         try {
 180             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 181                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 182         } catch (ExecutionException | InterruptedException e) {
 183             LOG.error("initilizeCaches: Exception while reading the DesignatedSwitchesForExternalTunnels DS", e);
 184             return;
 185         }
 186         if (designatedSwitchForTunnelOptional.isPresent()) {
 187             List<DesignatedSwitchForTunnel> list =
 188                 designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 189             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 190                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 191                         designatedDpnsToTunnelIpElanNameCache
 192                                 .get(Uint64.valueOf(designatedSwitchForTunnel.getDpId()));
 193                 if (setOfTunnelIpElanNamePair == null) {
 194                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 195                 }
 196                 Pair<IpAddress, String> tunnelIpElanNamePair =
 197                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 198                                 designatedSwitchForTunnel.getElanInstanceName());
 199                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 200                 designatedDpnsToTunnelIpElanNameCache.put(Uint64.valueOf(designatedSwitchForTunnel.getDpId()),
 201                         setOfTunnelIpElanNamePair);
 202             }
 203         }
 204         LOG.trace("Loading vniMacAddressToPortCache");
 205         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 206         Optional<Ports> optionalPorts;
 207         try {
 208             optionalPorts = SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION,
 209                     inst);
 210         } catch (ExecutionException | InterruptedException e) {
 211             LOG.error("initilizeCaches: Exception while reading the Ports DS", e);
 212             return;
 213         }
 214         if (optionalPorts.isPresent()) {
 215             List<Port> list = optionalPorts.get().nonnullPort();
 216             for (Port port : list) {
 217                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 218                     continue;
 219                 }
 220                 String macAddress = port.getMacAddress().getValue();
 221                 Uuid networkId = port.getNetworkId();
 222                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 223                 if (segmentationId == null) {
 224                     return;
 225                 }
 226                 updateVniMacToPortCache(Uint64.valueOf(new BigInteger(segmentationId)), macAddress, port);
 227             }
 228         }
 229     }
 230
 231     public Uint64 designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<Uint64> dpns) {
 232         Uint64 designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 233         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 234             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 235                     elanInstanceName);
 236             return designatedDpnId;
 237         }
 238         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 239     }
 240
 241     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<Uint64> dpns,
 242             final Uint64 designatedDpnId, final String vmMacAddress) {
 243         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 244                 elanInstanceName, designatedDpnId, vmMacAddress);
 245
 246         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 247         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 248             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 249                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 250                 return Collections.singletonList(
 251                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 252                         dpns.remove(designatedDpnId);
 253                         for (Uint64 dpn : dpns) {
 254                             installDhcpDropAction(dpn, vmMacAddress, tx);
 255                         }
 256                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 257                     }));
 258             } else {
 259                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 260             }
 261
 262             return Collections.emptyList();
 263         });
 264
 265         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 266     }
 267
 268     public void installDhcpFlowsForVms(Uint64 designatedDpnId, Set<String> listVmMacAddress,
 269             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 270         for (String vmMacAddress : listVmMacAddress) {
 271             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 272         }
 273     }
 274
 275     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<Uint64> dpns, String vmMacAddress) {
 276         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 277         removeFromLocalCache(elanInstanceName, vmMacAddress);
 278     }
 279
 280     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<Uint64> dpns) {
 281         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 282         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 283         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 284                 elanInstanceName, tunnelIp, dpns, vmMacs);
 285         if (vmMacs == null) {
 286             return;
 287         }
 288         for (String vmMacAddress : vmMacs) {
 289             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 290         }
 291         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 292     }
 293
 294     @NonNull
 295     public Uint64 readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 296         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 297             return Uint64.ZERO;
 298         }
 299         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 300                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 301                         .child(DesignatedSwitchForTunnel.class,
 302                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 303         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional;
 304         try {
 305             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 306                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 307         } catch (ExecutionException | InterruptedException e) {
 308             LOG.error("readDesignatedSwitchesForExternalTunnel: Exception while reading the DesignatedSwitchForTunnel "
 309                     + "DS for the elan-instance {} tunnelIp {}", elanInstanceName, tunnelIp, e);
 310             return Uint64.ZERO;
 311         }
 312         if (designatedSwitchForTunnelOptional.isPresent()) {
 313             return Uint64.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 314         }
 315         return Uint64.ZERO;
 316     }
 317
 318     public void writeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 319                                                        String elanInstanceName) {
 320         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 321                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 322         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 323                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 324                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 325         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 326                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 327                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 328                         .withKey(designatedSwitchForTunnelKey).build();
 329         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 330                 dpnId);
 331         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 332         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 333     }
 334
 335     public void removeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 336                                                         String elanInstanceName) {
 337         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 338                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 339         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 340                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 341                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 342         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 343                 elanInstanceName, dpnId);
 344         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 345         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 346     }
 347
 348     // This method is called whenever new OVS Switch is added.
 349     public void installDhcpDropActionOnDpn(Uint64 dpId) {
 350         // During controller restart we'll get add for designatedDpns as well and we
 351         // need not install drop flows for those dpns
 352         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 353             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 354             return;
 355         }
 356         // Read from DS since the cache may not get loaded completely in restart scenario
 357         if (isDpnDesignatedDpn(dpId)) {
 358             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 359             return;
 360         }
 361         List<String> vmMacs = getAllVmMacs();
 362         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 363         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 364             for (String vmMacAddress : vmMacs) {
 365                 installDhcpDropAction(dpId, vmMacAddress, tx);
 366             }
 367         }), LOG, "Error writing to the datastore");
 368     }
 369
 370     private boolean isDpnDesignatedDpn(Uint64 dpId) {
 371         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 372                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 373         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional;
 374         try {
 375             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 376                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 377         } catch (ExecutionException | InterruptedException e) {
 378             LOG.error("isDpnDesignatedDpn: Exception while reading the DesignatedSwitchesForExternalTunnels "
 379                     + "DS for the dpId {}", dpId, e);
 380             return false;
 381         }
 382         if (designatedSwitchForTunnelOptional.isPresent()) {
 383             List<DesignatedSwitchForTunnel> list =
 384                     designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 385             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 386                 if (dpId.equals(Uint64.valueOf(designatedSwitchForTunnel.getDpId()))) {
 387                     return true;
 388                 }
 389             }
 390         }
 391         return false;
 392     }
 393
 394     private List<String> getAllVmMacs() {
 395         List<String> vmMacs = new LinkedList<>();
 396         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 397         for (Set<String> list : listOfVmMacs) {
 398             vmMacs.addAll(list);
 399         }
 400         return vmMacs;
 401     }
 402
 403     public void updateLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 404         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 405         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 406         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 407         if (tunnelIpElanNameSet == null) {
 408             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 409         }
 410         tunnelIpElanNameSet.add(tunnelIpElanName);
 411         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 412                 tunnelIpElanNameSet);
 413         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 414     }
 415
 416     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 417         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 418         Set<String> setOfExistingVmMacAddress;
 419         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 420         if (setOfExistingVmMacAddress == null) {
 421             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 422         }
 423         setOfExistingVmMacAddress.add(vmMacAddress);
 424         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 425                 setOfExistingVmMacAddress);
 426         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 427         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 428     }
 429
 430     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 431         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 432         Set<String> listExistingVmMacAddress;
 433         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 434         if (listExistingVmMacAddress == null) {
 435             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 436         }
 437         listExistingVmMacAddress.add(vmMacAddress);
 438         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 439                 listExistingVmMacAddress);
 440         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 441     }
 442
 443     public void handleDesignatedDpnDown(Uint64 dpnId, List<Uint64> listOfDpns) {
 444         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 445         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 446         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 447             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 448         } else {
 449             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 450                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 451                     List<String> listOfVms = getAllVmMacs();
 452                     for (String vmMacAddress : listOfVms) {
 453                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 454                     }
 455                 }
 456                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 457                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 458                 }
 459             }), LOG, "Error writing to datastore");
 460         }
 461     }
 462
 463     public void updateCacheAndInstallNewFlows(List<Uint64> listOfDpns, Pair<IpAddress, String> pair,
 464             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 465         Uint64 newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 466         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 467             return;
 468         }
 469         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 470         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 471             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 472             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 473         }
 474         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 475         if (subnetDhcpData.isPresent()) {
 476             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 477                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 478         }
 479     }
 480
 481     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, Uint64 dpnId,
 482                                           TypedReadWriteTransaction<Configuration> tx)
 483             throws ExecutionException, InterruptedException {
 484         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 485         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 486             return;
 487         }
 488         for (String vmMacAddress : setOfVmMacAddress) {
 489             installDhcpDropAction(dpnId, vmMacAddress, tx);
 490         }
 491     }
 492
 493     /**
 494      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 495      * @param tunnelIp The tunnel Ip address
 496      * @param elanInstanceName The elan instance name
 497      * @param dpns The data path nodes
 498      * @return The designated dpn
 499      */
 500     private Uint64 chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 501             List<Uint64> dpns) {
 502         Uint64 designatedDpnId = DhcpMConstants.INVALID_DPID;
 503         if (dpns != null && dpns.size() != 0) {
 504             List<Uint64> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 505             candidateDpns.retainAll(dpns);
 506             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 507                     tunnelIp, elanInstanceName, candidateDpns);
 508             boolean elanDpnAvailableFlag = true;
 509             if (candidateDpns.isEmpty()) {
 510                 candidateDpns = dpns;
 511                 elanDpnAvailableFlag = false;
 512             }
 513             int size = 0;
 514             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 515             if (device == null) {
 516                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 517                         elanInstanceName, tunnelIp);
 518                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 519                 return designatedDpnId;
 520             }
 521             for (Uint64 dpn : candidateDpns) {
 522                 String hwvtepNodeId = device.getHwvtepNodeId();
 523                 if (!elanDpnAvailableFlag) {
 524                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 525                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 526                         continue;
 527                     }
 528                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 529                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 530                     continue;
 531                 }
 532                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 533                 if (tunnelIpElanNameSet == null) {
 534                     designatedDpnId = dpn;
 535                     break;
 536                 }
 537                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 538                     size = tunnelIpElanNameSet.size();
 539                     designatedDpnId = dpn;
 540                 }
 541             }
 542             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 543             return designatedDpnId;
 544         }
 545         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 546         return designatedDpnId;
 547     }
 548
 549     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 550         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 551     }
 552
 553     private void installDhcpEntries(Uint64 dpnId, String vmMacAddress,
 554             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 555         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 556                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 557     }
 558
 559     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 560                                               String dhcpMacAddress) {
 561         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 562         for (Entry<Uint64, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 563             Uint64 dpn = entry.getKey();
 564             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 565             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 566                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 567                     if (addFlow) {
 568                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 569                                 elanInstanceName, pair.getLeft());
 570                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 571                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 572                     } else {
 573                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 574                                 elanInstanceName, pair.getLeft());
 575                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 576                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 577                     }
 578                 }
 579             }
 580         }
 581     }
 582
 583
 584     public void configureDhcpArpRequestResponseFlow(Uint64 dpnId, String elanInstanceName, boolean addFlow,
 585                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 586         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 587         if (device == null) {
 588             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 589                     elanInstanceName);
 590         }
 591         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 592             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 593                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 594                 String tunnelInterfaceName = getExternalTunnelInterfaceName(dpnId.toString(),
 595                         device.getHwvtepNodeId());
 596                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 597                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 598                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 599                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 600                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 601                 if (optElan.isPresent()) {
 602                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 603                             lportTag, optElan.get().getElanTag());
 604                     Uuid nwUuid = new Uuid(elanInstanceName);
 605                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 606                     Uint64 vni = strVni != null ? Uint64.valueOf(strVni) : Uint64.valueOf(0);
 607                     if (!vni.equals(Uint64.ZERO)) {
 608                         return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 609                             Datastore.CONFIGURATION, tx -> {
 610                                 if (addFlow) {
 611                                     LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 612                                         dpnId, dhcpIpAddress, lportTag);
 613                                     installDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag,
 614                                         optElan.get().getElanTag().toJava());
 615                                     installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 616                                         dhcpIpAddress, dhcpMacAddress);
 617                                 } else {
 618                                     LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 619                                         dpnId, dhcpIpAddress, lportTag);
 620                                     uninstallDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag);
 621                                     uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 622                                 }
 623                             }));
 624                     }
 625                 }
 626             }
 627             return Collections.emptyList();
 628         });
 629     }
 630
 631     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 632         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 633         Uuid nwUuid = new Uuid(elanInstanceName);
 634         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 635         for (Uuid subnet : subnets) {
 636             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 637                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 638                 return optSubnetDhcp;
 639             }
 640         }
 641         return optSubnetDhcp;
 642     }
 643
 644     private void installDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 645                                             Uint64 vni, String dhcpIpAddress, int lportTag, Long elanTag)
 646             throws ExecutionException, InterruptedException {
 647         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 648                 lportTag, elanTag, true, mdsalUtil, tx);
 649     }
 650
 651     private void installDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 652                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 653         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 654                 elanInstanceName, lportTag, dhcpIpAddress);
 655         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 656         builder.setDpId(dpnId.toJava()).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 657                 .setLportTag(lportTag);
 658         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 659                 dhcpIpAddress, dhcpMacAddress, itmRpcService));
 660         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 661     }
 662
 663     private void uninstallDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 664                                                 String dhcpIpAddress) {
 665         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 666         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId.toJava())
 667                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 668                 .setLportTag(lportTag).buildForRemoveFlow();
 669         elanService.removeArpResponderFlow(arpInput);
 670     }
 671
 672     private void uninstallDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 673                                               Uint64 vni, String dhcpIpAddress, int lportTag)
 674             throws ExecutionException, InterruptedException {
 675         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 676                 lportTag, null, false, mdsalUtil, tx);
 677     }
 678
 679
 680     public void unInstallDhcpEntries(Uint64 dpnId, String vmMacAddress,
 681             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 682         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 683                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 684     }
 685
 686     private void installDhcpDropAction(Uint64 dpn, String vmMacAddress,
 687             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 688         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 689                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 690     }
 691
 692     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, Uint64 interfaceDpn) {
 693         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 694         if (interfaceDpn == null) {
 695             return Collections.emptyList();
 696         }
 697         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 698             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 699                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 700             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 701                 return Collections.emptyList();
 702             }
 703             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 704                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 705                 if (tunnelIpInDpn.equals(tunnelIp)) {
 706                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 707                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 708                                 tunnelElanPair);
 709                         return Collections.emptyList();
 710                     }
 711                 }
 712             }
 713             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 714                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 715                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 716                     String elanInstanceName = tunnelElanPair.getRight();
 717                     if (tunnelIpInDpn.equals(tunnelIp)) {
 718                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 719                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 720                                     tunnelElanPair);
 721                         }
 722                         List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 723                         dpns.remove(interfaceDpn);
 724                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 725                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 726                         if (subnetDhcpData.isPresent()) {
 727                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 728                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 729                                     subnetDhcpData.get().getPortMacaddress());
 730                         }
 731                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 732                     }
 733                 }
 734             }));
 735         }
 736     }
 737
 738     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 739         for (L2GatewayDevice device : ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight())) {
 740             if (Objects.equals(device.getTunnelIp(), tunnelElanPair.getLeft())) {
 741                 return true;
 742             }
 743         }
 744         return false;
 745     }
 746
 747     private String getTunnelIpDpnKey(IpAddress tunnelIp, Uint64 interfaceDpn) {
 748         return tunnelIp.toString() + interfaceDpn;
 749     }
 750
 751     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 752         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 753             Pair<IpAddress, String> pair = entry.getKey();
 754             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 755                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 756                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 757                     continue;
 758                 }
 759                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 760                         setOfExistingVmMacAddress, elanInstanceName);
 761                 setOfExistingVmMacAddress.remove(vmMacAddress);
 762                 if (setOfExistingVmMacAddress.size() > 0) {
 763                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 764                     return;
 765                 }
 766                 tunnelIpElanNameToVmMacCache.remove(pair);
 767             }
 768         }
 769     }
 770
 771     public void removeFromLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 772         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 773         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 774         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 775         if (tunnelIpElanNameSet != null) {
 776             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 777                             + "designatedDpnId {}",
 778                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 779             tunnelIpElanNameSet.remove(tunnelIpElanName);
 780             if (tunnelIpElanNameSet.size() != 0) {
 781                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 782             } else {
 783                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 784             }
 785         }
 786     }
 787
 788     public void updateVniMacToPortCache(Uint64 vni, String macAddress, Port port) {
 789         if (macAddress == null) {
 790             return;
 791         }
 792         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 793                 vni, macAddress.toUpperCase(Locale.getDefault()));
 794         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 795                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 796         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 797     }
 798
 799     public void removeVniMacToPortCache(Uint64 vni, String macAddress) {
 800         if (macAddress == null) {
 801             return;
 802         }
 803         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 804                 vni, macAddress.toUpperCase(Locale.getDefault()));
 805         vniMacAddressToPortCache.remove(vniMacAddressPair);
 806     }
 807
 808     @Nullable
 809     public Port readVniMacToPortCache(Uint64 vni, String macAddress) {
 810         if (macAddress == null) {
 811             return null;
 812         }
 813         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 814                 vni, macAddress.toUpperCase(Locale.getDefault()));
 815         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 816                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 817                 vniMacAddressToPortCache.get(vniMacAddressPair));
 818         return vniMacAddressToPortCache.get(vniMacAddressPair);
 819     }
 820
 821     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 822         String tunnelInterfaceName = null;
 823         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 824         try {
 825             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 826                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 827                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 828
 829             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 830             if (rpcResult.isSuccessful()) {
 831                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 832                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 833             } else {
 834                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 835             }
 836         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 837             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 838                     sourceNode, dstNode, e);
 839         }
 840         return tunnelInterfaceName;
 841     }
 842
 843     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 844         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 845                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 846         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 847     }
 848
 849     @Nullable
 850     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 851         Set<LocatorSet> locators = new HashSet<>();
 852         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 853                 internalTunnelIp.getIpv4Address().getValue());
 854         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 855                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 856                         terminationPointKey));
 857         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 858
 859         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 860                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 861
 862         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 863                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 864                 .setLogicalSwitchRef(lsRef).build();
 865         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 866                 dstDevice.getNodeId(), remoteMcastMacs.key());
 867         ReadTransaction transaction = broker.newReadOnlyTransaction();
 868         try {
 869             //TODO do async mdsal read
 870             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).get().get();
 871             locators.addAll(remoteMcastMacs.getLocatorSet());
 872             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 873         } catch (InterruptedException | ExecutionException e) {
 874             LOG.error("Failed to read the macs {}", iid);
 875         } finally {
 876             transaction.close();
 877         }
 878         return null;
 879     }
 880
 881     private void putRemoteMcastMac(TypedWriteTransaction<Configuration> transaction, String elanName,
 882                                    L2GatewayDevice device, IpAddress internalTunnelIp) {
 883         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 884         if (!optionalNode.isPresent()) {
 885             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 886             return;
 887         }
 888         Node dstNode = optionalNode.get();
 889         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 890         HwvtepUtils.addRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 891     }
 892
 893     public void installRemoteMcastMac(final Uint64 designatedDpnId, final IpAddress tunnelIp,
 894                                       final String elanInstanceName) {
 895         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 896             return;
 897         }
 898
 899         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 900             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 901                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 902                 LOG.info("Installing remote McastMac is not executed for this node.");
 903                 return Collections.emptyList();
 904             }
 905
 906             LOG.info("Installing remote McastMac");
 907             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 908             if (device == null) {
 909                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 910                     elanInstanceName);
 911                 return Collections.emptyList();
 912             }
 913             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 914                     device.getHwvtepNodeId());
 915             if (tunnelInterfaceName != null) {
 916                 Interface tunnelInterface =
 917                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 918                 if (tunnelInterface == null) {
 919                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 920                     return Collections.emptyList();
 921                 }
 922                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 923                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 924                             tunnelInterface.augmentation(IfTunnel.class).getTunnelSource())));
 925             }
 926             return Collections.emptyList();
 927         });
 928     }
 929
 930     @Nullable
 931     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 932         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 933         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 934         for (L2GatewayDevice device : devices) {
 935             if (tunnelIp.equals(device.getTunnelIp())) {
 936                 return device;
 937             }
 938         }
 939         return null;
 940     }
 941
 942     private boolean isTunnelUp(String nodeName, Uint64 dpn) {
 943         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 944         if (tunnelInterfaceName == null) {
 945             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 946             return false;
 947         }
 948         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 949                 .Interface tunnelInterface =
 950                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 951         if (tunnelInterface == null) {
 952             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 953             return false;
 954         }
 955         return tunnelInterface.getOperStatus() == OperStatus.Up;
 956     }
 957
 958     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, Uint64 interfaceDpn) {
 959         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 960         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 961             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 962                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 963             List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 964             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 965                 LOG.trace("There are no undesignated DPNs");
 966                 return Collections.emptyList();
 967             }
 968             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 969                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 970                     if (tunnelIp.equals(pair.getLeft())) {
 971                         String elanInstanceName = pair.getRight();
 972                         Uint64 newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 973                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 974                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 975                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 976                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 977                                         vmMacAddress, newDesignatedDpn);
 978                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 979                             }
 980                         }
 981                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 982                         if (subnetDhcpData.isPresent()) {
 983                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 984                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 985                                     subnetDhcpData.get().getPortMacaddress());
 986                         }
 987                     }
 988                 }
 989             }));
 990         }
 991     }
 992
 993     private boolean isTunnelConfigured(Uint64 dpn, String hwVtepNodeId) {
 994         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 995         if (tunnelInterfaceName == null) {
 996             return false;
 997         }
 998         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 999         if (tunnelInterface == null) {
1000             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
1001             return false;
1002         }
1003         return true;
1004     }
1005
1006     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
1007         availableVMCache.remove(tunnelIpElanName);
1008     }
1009
1010     private void unInstallDhcpEntriesOnDpns(final List<Uint64> dpns, final String vmMacAddress) {
1011         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
1012             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
1013                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
1014                 return Collections.singletonList(
1015                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1016                         for (final Uint64 dpn : dpns) {
1017                             unInstallDhcpEntries(dpn, vmMacAddress, tx);
1018                         }
1019                     }));
1020             } else {
1021                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
1022             }
1023
1024             return Collections.emptyList();
1025         });
1026     }
1027
1028     @Nullable
1029     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
1030         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
1031         IpAddress tunnelIp = null;
1032         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
1033             Pair<IpAddress, String> pair = entry.getKey();
1034             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
1035                     pair.getRight());
1036             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
1037                 Set<String> listExistingVmMacAddress = entry.getValue();
1038                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
1039                         && listExistingVmMacAddress.contains(vmMacAddress)) {
1040                     tunnelIp = pair.getLeft();
1041                     break;
1042                 }
1043             }
1044         }
1045         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
1046         return tunnelIp;
1047     }
1048
1049     private String getJobKey(final String jobKeySuffix) {
1050         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
1051     }
1052 }