dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.util.concurrent.ListenableFuture;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.HashSet;
  17 import java.util.LinkedList;
  18 import java.util.List;
  19 import java.util.Locale;
  20 import java.util.Map.Entry;
  21 import java.util.Set;
  22 import java.util.concurrent.ConcurrentHashMap;
  23 import java.util.concurrent.ConcurrentMap;
  24 import java.util.concurrent.CopyOnWriteArraySet;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.annotation.PostConstruct;
  28 import javax.inject.Inject;
  29 import javax.inject.Named;
  30 import javax.inject.Singleton;
  31 import org.apache.commons.lang3.tuple.ImmutablePair;
  32 import org.apache.commons.lang3.tuple.Pair;
  33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  34 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  45 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  46 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  47 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  50 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  51 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  52 import org.opendaylight.netvirt.dhcpservice.api.IDhcpExternalTunnelManager;
  53 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  54 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  55 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  56 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  57 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  58 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  59 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  90 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
  91 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
  92 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
  93 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  94 import org.opendaylight.yangtools.yang.common.RpcResult;
  95 import org.slf4j.Logger;
  96 import org.slf4j.LoggerFactory;
  97
  98 @Singleton
  99 public class DhcpExternalTunnelManager implements IDhcpExternalTunnelManager {
 100
 101     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 102     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 103
 104     private final DataBroker broker;
 105     private final ManagedNewTransactionRunner txRunner;
 106     private final IMdsalApiManager mdsalUtil;
 107     private final ItmRpcService itmRpcService;
 108     private final EntityOwnershipUtils entityOwnershipUtils;
 109     private final IInterfaceManager interfaceManager;
 110     private final JobCoordinator jobCoordinator;
 111     private final L2GatewayCache l2GatewayCache;
 112     private IElanService elanService;
 113
 114     private final ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 115             new ConcurrentHashMap<>();
 116     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 117             new ConcurrentHashMap<>();
 118     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 119     private final ConcurrentMap<Pair<BigInteger, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 120
 121     @Override
 122     public ConcurrentMap<BigInteger, Set<Pair<IpAddress, String>>> getDesignatedDpnsToTunnelIpElanNameCache() {
 123         return designatedDpnsToTunnelIpElanNameCache;
 124     }
 125
 126     @Override
 127     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getTunnelIpElanNameToVmMacCache() {
 128         return tunnelIpElanNameToVmMacCache;
 129     }
 130
 131     @Override
 132     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getAvailableVMCache() {
 133         return availableVMCache;
 134     }
 135
 136     @Override
 137     public ConcurrentMap<Pair<BigInteger, String>, Port> getVniMacAddressToPortCache() {
 138         return vniMacAddressToPortCache;
 139     }
 140
 141     @Inject
 142     public DhcpExternalTunnelManager(final DataBroker broker,
 143             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 144             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 145             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 146             @Named("elanService") IElanService ielanService) {
 147         this.broker = broker;
 148         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 149         this.mdsalUtil = mdsalUtil;
 150         this.itmRpcService = itmRpcService;
 151         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 152         this.interfaceManager = interfaceManager;
 153         this.jobCoordinator = jobCoordinator;
 154         this.l2GatewayCache = l2GatewayCache;
 155         this.elanService = ielanService;
 156     }
 157
 158     @PostConstruct
 159     public void init() {
 160         initilizeCaches();
 161     }
 162
 163     private void initilizeCaches() {
 164         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 165         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 166                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 167         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 168                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 169         if (designatedSwitchForTunnelOptional.isPresent()) {
 170             List<DesignatedSwitchForTunnel> list =
 171                     designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 172             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 173                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 174                         designatedDpnsToTunnelIpElanNameCache
 175                                 .get(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()));
 176                 if (setOfTunnelIpElanNamePair == null) {
 177                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 178                 }
 179                 Pair<IpAddress, String> tunnelIpElanNamePair =
 180                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 181                                 designatedSwitchForTunnel.getElanInstanceName());
 182                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 183                 designatedDpnsToTunnelIpElanNameCache.put(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()),
 184                         setOfTunnelIpElanNamePair);
 185             }
 186         }
 187         LOG.trace("Loading vniMacAddressToPortCache");
 188         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 189         Optional<Ports> optionalPorts = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, inst);
 190         if (optionalPorts.isPresent()) {
 191             List<Port> list = optionalPorts.get().getPort();
 192             for (Port port : list) {
 193                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 194                     continue;
 195                 }
 196                 String macAddress = port.getMacAddress().getValue();
 197                 Uuid networkId = port.getNetworkId();
 198                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 199                 if (segmentationId == null) {
 200                     return;
 201                 }
 202                 updateVniMacToPortCache(new BigInteger(segmentationId), macAddress, port);
 203             }
 204         }
 205     }
 206
 207     public BigInteger designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<BigInteger> dpns) {
 208         BigInteger designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 209         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 210             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 211                     elanInstanceName);
 212             return designatedDpnId;
 213         }
 214         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 215     }
 216
 217     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<BigInteger> dpns,
 218             final BigInteger designatedDpnId, final String vmMacAddress) {
 219         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 220                 elanInstanceName, designatedDpnId, vmMacAddress);
 221
 222         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 223         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 224             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 225                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 226                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 227                     dpns.remove(designatedDpnId);
 228                     for (BigInteger dpn : dpns) {
 229                         installDhcpDropAction(dpn, vmMacAddress, tx);
 230                     }
 231                     installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 232                 }));
 233             } else {
 234                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 235             }
 236
 237             return Collections.emptyList();
 238         });
 239
 240         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 241     }
 242
 243     public void installDhcpFlowsForVms(BigInteger designatedDpnId, Set<String> listVmMacAddress, WriteTransaction tx) {
 244         for (String vmMacAddress : listVmMacAddress) {
 245             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 246         }
 247     }
 248
 249     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<BigInteger> dpns, String vmMacAddress) {
 250         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 251         removeFromLocalCache(elanInstanceName, vmMacAddress);
 252     }
 253
 254     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<BigInteger> dpns) {
 255         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 256         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 257         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 258                 elanInstanceName, tunnelIp, dpns, vmMacs);
 259         if (vmMacs == null) {
 260             return;
 261         }
 262         for (String vmMacAddress : vmMacs) {
 263             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 264         }
 265         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 266     }
 267
 268     public BigInteger readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 269         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 270             return null;
 271         }
 272         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 273                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 274                         .child(DesignatedSwitchForTunnel.class,
 275                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 276         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional =
 277                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 278         if (designatedSwitchForTunnelOptional.isPresent()) {
 279             return BigInteger.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 280         }
 281         return null;
 282     }
 283
 284     public void writeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 285                                                        String elanInstanceName) {
 286         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 287                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 288         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 289                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 290                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 291         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 292                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 293                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 294                         .withKey(designatedSwitchForTunnelKey).build();
 295         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 296                 dpnId);
 297         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 298         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 299     }
 300
 301     public void removeDesignatedSwitchForExternalTunnel(BigInteger dpnId, IpAddress tunnelIp,
 302                                                         String elanInstanceName) {
 303         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 304                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 305         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 306                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 307                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 308         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 309                 elanInstanceName, dpnId);
 310         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 311         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 312     }
 313
 314     // This method is called whenever new OVS Switch is added.
 315     public void installDhcpDropActionOnDpn(BigInteger dpId) {
 316         // During controller restart we'll get add for designatedDpns as well and we
 317         // need not install drop flows for those dpns
 318         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 319             LOG.trace("The dpn {} is designated DPN need not install drop flows");
 320             return;
 321         }
 322         // Read from DS since the cache may not get loaded completely in restart scenario
 323         if (isDpnDesignatedDpn(dpId)) {
 324             LOG.trace("The dpn {} is designated DPN need not install drop flows");
 325             return;
 326         }
 327         List<String> vmMacs = getAllVmMacs();
 328         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 329         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 330             for (String vmMacAddress : vmMacs) {
 331                 installDhcpDropAction(dpId, vmMacAddress, tx);
 332             }
 333         }), LOG, "Error writing to the datastore");
 334     }
 335
 336     private boolean isDpnDesignatedDpn(BigInteger dpId) {
 337         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 338                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 339         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional =
 340                 MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 341         if (designatedSwitchForTunnelOptional.isPresent()) {
 342             List<DesignatedSwitchForTunnel> list =
 343                     designatedSwitchForTunnelOptional.get().getDesignatedSwitchForTunnel();
 344             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : list) {
 345                 if (dpId.equals(BigInteger.valueOf(designatedSwitchForTunnel.getDpId()))) {
 346                     return true;
 347                 }
 348             }
 349         }
 350         return false;
 351     }
 352
 353     private List<String> getAllVmMacs() {
 354         List<String> vmMacs = new LinkedList<>();
 355         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 356         for (Set<String> list : listOfVmMacs) {
 357             vmMacs.addAll(list);
 358         }
 359         return vmMacs;
 360     }
 361
 362     public void updateLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 363         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 364         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 365         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 366         if (tunnelIpElanNameSet == null) {
 367             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 368         }
 369         tunnelIpElanNameSet.add(tunnelIpElanName);
 370         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 371                 tunnelIpElanNameSet);
 372         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 373     }
 374
 375     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 376         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 377         Set<String> setOfExistingVmMacAddress;
 378         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 379         if (setOfExistingVmMacAddress == null) {
 380             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 381         }
 382         setOfExistingVmMacAddress.add(vmMacAddress);
 383         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 384                 setOfExistingVmMacAddress);
 385         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 386         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 387     }
 388
 389     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 390         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 391         Set<String> listExistingVmMacAddress;
 392         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 393         if (listExistingVmMacAddress == null) {
 394             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 395         }
 396         listExistingVmMacAddress.add(vmMacAddress);
 397         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 398                 listExistingVmMacAddress);
 399         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 400     }
 401
 402     public void handleDesignatedDpnDown(BigInteger dpnId, List<BigInteger> listOfDpns) {
 403         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 404         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 405         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 406             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 407         } else {
 408             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 409                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 410                     List<String> listOfVms = getAllVmMacs();
 411                     for (String vmMacAddress : listOfVms) {
 412                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 413                     }
 414                 }
 415                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 416                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 417                 }
 418             }), LOG, "Error writing to datastore");
 419         }
 420     }
 421
 422     public void updateCacheAndInstallNewFlows(List<BigInteger> listOfDpns, Pair<IpAddress, String> pair,
 423             WriteTransaction tx) {
 424         BigInteger newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 425         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 426             return;
 427         }
 428         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 429         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 430             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 431             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 432         }
 433         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 434         if (subnetDhcpData.isPresent()) {
 435             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 436                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 437         }
 438     }
 439
 440     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, BigInteger dpnId,
 441                                           WriteTransaction tx) {
 442         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 443         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 444             return;
 445         }
 446         for (String vmMacAddress : setOfVmMacAddress) {
 447             installDhcpDropAction(dpnId, vmMacAddress, tx);
 448         }
 449     }
 450
 451     /**
 452      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 453      * @param tunnelIp The tunnel Ip address
 454      * @param elanInstanceName The elan instance name
 455      * @param dpns The data path nodes
 456      * @return The designated dpn
 457      */
 458     private BigInteger chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 459             List<BigInteger> dpns) {
 460         BigInteger designatedDpnId = DhcpMConstants.INVALID_DPID;
 461         if (dpns != null && dpns.size() != 0) {
 462             List<BigInteger> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 463             candidateDpns.retainAll(dpns);
 464             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 465                     tunnelIp, elanInstanceName, candidateDpns);
 466             boolean elanDpnAvailableFlag = true;
 467             if (candidateDpns.isEmpty()) {
 468                 candidateDpns = dpns;
 469                 elanDpnAvailableFlag = false;
 470             }
 471             int size = 0;
 472             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 473             if (device == null) {
 474                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 475                         elanInstanceName, tunnelIp);
 476                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 477                 return designatedDpnId;
 478             }
 479             for (BigInteger dpn : candidateDpns) {
 480                 String hwvtepNodeId = device.getHwvtepNodeId();
 481                 if (!elanDpnAvailableFlag) {
 482                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 483                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 484                         continue;
 485                     }
 486                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 487                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 488                     continue;
 489                 }
 490                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 491                 if (tunnelIpElanNameSet == null) {
 492                     designatedDpnId = dpn;
 493                     break;
 494                 }
 495                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 496                     size = tunnelIpElanNameSet.size();
 497                     designatedDpnId = dpn;
 498                 }
 499             }
 500             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 501             return designatedDpnId;
 502         }
 503         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 504         return designatedDpnId;
 505     }
 506
 507     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 508         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 509     }
 510
 511     private void installDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 512         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 513                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 514     }
 515
 516     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 517                                               String dhcpMacAddress) {
 518         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 519         for (Entry<BigInteger, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 520             BigInteger dpn = entry.getKey();
 521             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 522             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 523                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 524                     if (addFlow) {
 525                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 526                                 elanInstanceName, pair.getLeft());
 527                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 528                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 529                     } else {
 530                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 531                                 elanInstanceName, pair.getLeft());
 532                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 533                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 534                     }
 535                 }
 536             }
 537         }
 538     }
 539
 540
 541     public void configureDhcpArpRequestResponseFlow(BigInteger dpnId, String elanInstanceName, boolean addFlow,
 542                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 543         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 544         if (device == null) {
 545             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 546                     elanInstanceName);
 547         }
 548         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 549             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 550                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 551                 String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpnId),
 552                         device.getHwvtepNodeId());
 553                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 554                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 555                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 556                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 557                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 558                 if (optElan.isPresent()) {
 559                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 560                             lportTag, optElan.get().getElanTag());
 561                     Uuid nwUuid = new Uuid(elanInstanceName);
 562                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 563                     BigInteger vni = strVni != null ? new BigInteger(strVni) : BigInteger.ZERO;
 564                     if (!vni.equals(BigInteger.ZERO)) {
 565                         if (addFlow) {
 566                             LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 567                                     dpnId, dhcpIpAddress, lportTag);
 568                             installDhcpArpRequestFlows(dpnId, vni, dhcpIpAddress, lportTag,
 569                                     optElan.get().getElanTag());
 570                             installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 571                                     dhcpIpAddress, dhcpMacAddress);
 572                         } else {
 573                             LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 574                                     dpnId, dhcpIpAddress, lportTag);
 575                             uninstallDhcpArpRequestFlows(dpnId, vni, dhcpIpAddress, lportTag);
 576                             uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 577                         }
 578                     }
 579                 }
 580             }
 581             return null;
 582         });
 583     }
 584
 585     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 586         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 587         Uuid nwUuid = new Uuid(elanInstanceName);
 588         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 589         for (Uuid subnet : subnets) {
 590             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 591                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 592                 return optSubnetDhcp;
 593             }
 594         }
 595         return optSubnetDhcp;
 596     }
 597
 598     private void installDhcpArpRequestFlows(BigInteger dpnId, BigInteger vni, String dhcpIpAddress,
 599                                             int lportTag, Long elanTag) {
 600         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 601                 lportTag, elanTag, true, mdsalUtil);
 602     }
 603
 604     private void installDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 605                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 606         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 607                 elanInstanceName, lportTag, dhcpIpAddress);
 608         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 609         builder.setDpId(dpnId).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 610                 .setLportTag(lportTag);
 611         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 612                 dhcpIpAddress, dhcpMacAddress, itmRpcService));
 613         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 614     }
 615
 616     private void uninstallDhcpArpResponderFlows(BigInteger dpnId, String interfaceName, int lportTag,
 617                                                 String dhcpIpAddress) {
 618         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 619         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId)
 620                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 621                 .setLportTag(lportTag).buildForRemoveFlow();
 622         elanService.removeArpResponderFlow(arpInput);
 623     }
 624
 625     private void uninstallDhcpArpRequestFlows(BigInteger dpnId, BigInteger vni, String dhcpIpAddress,
 626                                               int lportTag) {
 627         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 628                 lportTag, null, false, mdsalUtil);
 629     }
 630
 631
 632     public void unInstallDhcpEntries(BigInteger dpnId, String vmMacAddress, WriteTransaction tx) {
 633         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 634                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, tx);
 635     }
 636
 637     private void installDhcpDropAction(BigInteger dpn, String vmMacAddress, WriteTransaction tx) {
 638         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 639                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, tx);
 640     }
 641
 642     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, BigInteger interfaceDpn) {
 643         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 644         if (interfaceDpn == null) {
 645             return Collections.emptyList();
 646         }
 647         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 648             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 649                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 650             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 651                 return Collections.emptyList();
 652             }
 653             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 654                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 655                 if (tunnelIpInDpn.equals(tunnelIp)) {
 656                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 657                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 658                                 tunnelElanPair);
 659                         return Collections.emptyList();
 660                     }
 661                 }
 662             }
 663             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 664                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 665                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 666                     String elanInstanceName = tunnelElanPair.getRight();
 667                     if (tunnelIpInDpn.equals(tunnelIp)) {
 668                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 669                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 670                                     tunnelElanPair);
 671                         }
 672                         List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 673                         dpns.remove(interfaceDpn);
 674                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 675                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 676                         if (subnetDhcpData.isPresent()) {
 677                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 678                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 679                                     subnetDhcpData.get().getPortMacaddress());
 680                         }
 681                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 682                     }
 683                 }
 684             }));
 685         }
 686     }
 687
 688     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 689         ConcurrentMap<String, L2GatewayDevice> l2GwDevices =
 690                 ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight());
 691         for (L2GatewayDevice device : l2GwDevices.values()) {
 692             if (device.getTunnelIp().equals(tunnelElanPair.getLeft())) {
 693                 return true;
 694             }
 695         }
 696         return false;
 697     }
 698
 699     private String getTunnelIpDpnKey(IpAddress tunnelIp, BigInteger interfaceDpn) {
 700         return tunnelIp.toString() + interfaceDpn;
 701     }
 702
 703     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 704         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 705             Pair<IpAddress, String> pair = entry.getKey();
 706             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 707                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 708                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 709                     continue;
 710                 }
 711                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 712                         setOfExistingVmMacAddress, elanInstanceName);
 713                 setOfExistingVmMacAddress.remove(vmMacAddress);
 714                 if (setOfExistingVmMacAddress.size() > 0) {
 715                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 716                     return;
 717                 }
 718                 tunnelIpElanNameToVmMacCache.remove(pair);
 719             }
 720         }
 721     }
 722
 723     public void removeFromLocalCache(BigInteger designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 724         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 725         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 726         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 727         if (tunnelIpElanNameSet != null) {
 728             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 729                             + "designatedDpnId {}",
 730                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 731             tunnelIpElanNameSet.remove(tunnelIpElanName);
 732             if (tunnelIpElanNameSet.size() != 0) {
 733                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 734             } else {
 735                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 736             }
 737         }
 738     }
 739
 740     public void updateVniMacToPortCache(BigInteger vni, String macAddress, Port port) {
 741         if (macAddress == null) {
 742             return;
 743         }
 744         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 745                 vni, macAddress.toUpperCase(Locale.getDefault()));
 746         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 747                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 748         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 749     }
 750
 751     public void removeVniMacToPortCache(BigInteger vni, String macAddress) {
 752         if (macAddress == null) {
 753             return;
 754         }
 755         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 756                 vni, macAddress.toUpperCase(Locale.getDefault()));
 757         vniMacAddressToPortCache.remove(vniMacAddressPair);
 758     }
 759
 760     public Port readVniMacToPortCache(BigInteger vni, String macAddress) {
 761         if (macAddress == null) {
 762             return null;
 763         }
 764         Pair<BigInteger, String> vniMacAddressPair = new ImmutablePair<>(
 765                 vni, macAddress.toUpperCase(Locale.getDefault()));
 766         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 767                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 768                 vniMacAddressToPortCache.get(vniMacAddressPair));
 769         return vniMacAddressToPortCache.get(vniMacAddressPair);
 770     }
 771
 772     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 773         String tunnelInterfaceName = null;
 774         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 775         try {
 776             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 777                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 778                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 779
 780             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 781             if (rpcResult.isSuccessful()) {
 782                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 783                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 784             } else {
 785                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 786             }
 787         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 788             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 789                     sourceNode, dstNode, e);
 790         }
 791         return tunnelInterfaceName;
 792     }
 793
 794     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 795         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 796                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 797         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 798     }
 799
 800     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 801         Set<LocatorSet> locators = new HashSet<>();
 802         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 803                 internalTunnelIp.getIpv4Address().getValue());
 804         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 805                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 806                         terminationPointKey));
 807         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 808
 809         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 810                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 811
 812         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 813                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 814                 .setLogicalSwitchRef(lsRef).build();
 815         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 816                 dstDevice.getNodeId(), remoteMcastMacs.key());
 817         ReadOnlyTransaction transaction = broker.newReadOnlyTransaction();
 818         try {
 819             //TODO do async mdsal read
 820             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).checkedGet().get();
 821             locators.addAll(remoteMcastMacs.getLocatorSet());
 822             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 823         } catch (ReadFailedException e) {
 824             LOG.error("Failed to read the macs {}", iid);
 825         }
 826         return null;
 827     }
 828
 829     private WriteTransaction putRemoteMcastMac(WriteTransaction transaction, String elanName,
 830                                                L2GatewayDevice device, IpAddress internalTunnelIp) {
 831         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 832         Node dstNode = optionalNode.get();
 833         if (dstNode == null) {
 834             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 835             return null;
 836         }
 837         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 838         HwvtepUtils.putRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 839         return transaction;
 840     }
 841
 842     public void installRemoteMcastMac(final BigInteger designatedDpnId, final IpAddress tunnelIp,
 843                                       final String elanInstanceName) {
 844         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 845             return;
 846         }
 847
 848         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 849             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 850                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 851                 LOG.info("Installing remote McastMac is not executed for this node.");
 852                 return Collections.emptyList();
 853             }
 854
 855             LOG.info("Installing remote McastMac");
 856             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 857             if (device == null) {
 858                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 859                     elanInstanceName);
 860                 return Collections.emptyList();
 861             }
 862             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 863                     device.getHwvtepNodeId());
 864             if (tunnelInterfaceName != null) {
 865                 Interface tunnelInterface =
 866                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 867                 if (tunnelInterface == null) {
 868                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 869                     return Collections.emptyList();
 870                 }
 871                 return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 872                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 873                             tunnelInterface.augmentation(IfTunnel.class).getTunnelSource())));
 874             }
 875             return Collections.emptyList();
 876         });
 877     }
 878
 879     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 880         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 881         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 882         for (L2GatewayDevice device : devices) {
 883             if (tunnelIp.equals(device.getTunnelIp())) {
 884                 return device;
 885             }
 886         }
 887         return null;
 888     }
 889
 890     private boolean isTunnelUp(String nodeName, BigInteger dpn) {
 891         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 892         if (tunnelInterfaceName == null) {
 893             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 894             return false;
 895         }
 896         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 897                 .Interface tunnelInterface =
 898                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 899         if (tunnelInterface == null) {
 900             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 901             return false;
 902         }
 903         return tunnelInterface.getOperStatus() == OperStatus.Up;
 904     }
 905
 906     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, BigInteger interfaceDpn) {
 907         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 908         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 909             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 910                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 911             List<BigInteger> dpns = DhcpServiceUtils.getListOfDpns(broker);
 912             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 913                 LOG.trace("There are no undesignated DPNs");
 914                 return Collections.emptyList();
 915             }
 916             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 917                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 918                     if (tunnelIp.equals(pair.getLeft())) {
 919                         String elanInstanceName = pair.getRight();
 920                         BigInteger newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 921                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 922                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 923                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 924                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 925                                         vmMacAddress, newDesignatedDpn);
 926                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 927                             }
 928                         }
 929                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 930                         if (subnetDhcpData.isPresent()) {
 931                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 932                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 933                                     subnetDhcpData.get().getPortMacaddress());
 934                         }
 935                     }
 936                 }
 937             }));
 938         }
 939     }
 940
 941     private boolean isTunnelConfigured(BigInteger dpn, String hwVtepNodeId) {
 942         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 943         if (tunnelInterfaceName == null) {
 944             return false;
 945         }
 946         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 947         if (tunnelInterface == null) {
 948             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 949             return false;
 950         }
 951         return true;
 952     }
 953
 954     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
 955         availableVMCache.remove(tunnelIpElanName);
 956     }
 957
 958     private void unInstallDhcpEntriesOnDpns(final List<BigInteger> dpns, final String vmMacAddress) {
 959         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
 960             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 961                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 962                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 963                     for (final BigInteger dpn : dpns) {
 964                         unInstallDhcpEntries(dpn, vmMacAddress, tx);
 965                     }
 966                 }));
 967             } else {
 968                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
 969             }
 970
 971             return Collections.emptyList();
 972         });
 973     }
 974
 975     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
 976         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
 977         IpAddress tunnelIp = null;
 978         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
 979             Pair<IpAddress, String> pair = entry.getKey();
 980             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
 981                     pair.getRight());
 982             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 983                 Set<String> listExistingVmMacAddress = entry.getValue();
 984                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
 985                         && listExistingVmMacAddress.contains(vmMacAddress)) {
 986                     tunnelIp = pair.getLeft();
 987                     break;
 988                 }
 989             }
 990         }
 991         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
 992         return tunnelIp;
 993     }
 994
 995     private String getJobKey(final String jobKeySuffix) {
 996         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
 997     }
 998 }