dhcpservice/impl/src/main/java/org/opendaylight/netvirt/dhcpservice/DhcpExternalTunnelManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.dhcpservice;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.util.concurrent.ListenableFuture;
  13 import java.math.BigInteger;
  14 import java.util.ArrayList;
  15 import java.util.Collection;
  16 import java.util.Collections;
  17 import java.util.HashSet;
  18 import java.util.LinkedList;
  19 import java.util.List;
  20 import java.util.Locale;
  21 import java.util.Map;
  22 import java.util.Map.Entry;
  23 import java.util.Objects;
  24 import java.util.Optional;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentMap;
  28 import java.util.concurrent.CopyOnWriteArraySet;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.Future;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Named;
  34 import javax.inject.Singleton;
  35 import org.apache.commons.lang3.tuple.ImmutablePair;
  36 import org.apache.commons.lang3.tuple.Pair;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.eclipse.jdt.annotation.Nullable;
  39 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  40 import org.opendaylight.genius.infra.Datastore;
  41 import org.opendaylight.genius.infra.Datastore.Configuration;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  44 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  45 import org.opendaylight.genius.infra.TypedWriteTransaction;
  46 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  50 import org.opendaylight.genius.utils.clustering.EntityOwnershipUtils;
  51 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  52 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  53 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  54 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  55 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  56 import org.opendaylight.mdsal.binding.api.DataBroker;
  57 import org.opendaylight.mdsal.binding.api.ReadTransaction;
  58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  59 import org.opendaylight.mdsal.eos.binding.api.EntityOwnershipService;
  60 import org.opendaylight.netvirt.dhcpservice.api.DhcpMConstants;
  61 import org.opendaylight.netvirt.dhcpservice.api.IDhcpExternalTunnelManager;
  62 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  63 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  64 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  65 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  66 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayCache;
  67 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  68 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  73 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.IfTunnel;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetExternalTunnelInterfaceNameOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.DesignatedSwitchesForExternalTunnels;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnel;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.dhcp.rev160428.designated.switches._for.external.tunnels.DesignatedSwitchForTunnelKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.Ports;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.PortKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.rev150712.Neutron;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.dhcpservice.api.rev150710.subnet.dhcp.port.data.SubnetToDhcpPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSetBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 100 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
 101 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPoint;
 102 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.node.TerminationPointKey;
 103 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 104 import org.opendaylight.yangtools.yang.common.RpcResult;
 105 import org.opendaylight.yangtools.yang.common.Uint64;
 106 import org.slf4j.Logger;
 107 import org.slf4j.LoggerFactory;
 108
 109 @Singleton
 110 public class DhcpExternalTunnelManager implements IDhcpExternalTunnelManager {
 111
 112     private static final Logger LOG = LoggerFactory.getLogger(DhcpExternalTunnelManager.class);
 113     public static final String UNKNOWN_DMAC = "00:00:00:00:00:00";
 114
 115     private final DataBroker broker;
 116     private final ManagedNewTransactionRunner txRunner;
 117     private final IMdsalApiManager mdsalUtil;
 118     private final ItmRpcService itmRpcService;
 119     private final EntityOwnershipUtils entityOwnershipUtils;
 120     private final IInterfaceManager interfaceManager;
 121     private final JobCoordinator jobCoordinator;
 122     private final L2GatewayCache l2GatewayCache;
 123     private IElanService elanService;
 124     private final DhcpServiceCounters dhcpServiceCounters;
 125
 126     private final ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> designatedDpnsToTunnelIpElanNameCache =
 127             new ConcurrentHashMap<>();
 128     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> tunnelIpElanNameToVmMacCache =
 129             new ConcurrentHashMap<>();
 130     private final ConcurrentMap<Pair<IpAddress, String>, Set<String>> availableVMCache = new ConcurrentHashMap<>();
 131     private final ConcurrentMap<Pair<Uint64, String>, Port> vniMacAddressToPortCache = new ConcurrentHashMap<>();
 132
 133     @Override
 134     public ConcurrentMap<Uint64, Set<Pair<IpAddress, String>>> getDesignatedDpnsToTunnelIpElanNameCache() {
 135         return designatedDpnsToTunnelIpElanNameCache;
 136     }
 137
 138     @Override
 139     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getTunnelIpElanNameToVmMacCache() {
 140         return tunnelIpElanNameToVmMacCache;
 141     }
 142
 143     @Override
 144     public ConcurrentMap<Pair<IpAddress, String>, Set<String>> getAvailableVMCache() {
 145         return availableVMCache;
 146     }
 147
 148     @Override
 149     public ConcurrentMap<Pair<Uint64, String>, Port> getVniMacAddressToPortCache() {
 150         return vniMacAddressToPortCache;
 151     }
 152
 153     @Inject
 154     public DhcpExternalTunnelManager(final DataBroker broker,
 155             final IMdsalApiManager mdsalUtil, final ItmRpcService itmRpcService,
 156             final EntityOwnershipService entityOwnershipService, final IInterfaceManager interfaceManager,
 157             final JobCoordinator jobCoordinator, final L2GatewayCache l2GatewayCache,
 158             @Named("elanService") IElanService ielanService, DhcpServiceCounters dhcpServiceCounters) {
 159         this.broker = broker;
 160         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 161         this.mdsalUtil = mdsalUtil;
 162         this.itmRpcService = itmRpcService;
 163         this.entityOwnershipUtils = new EntityOwnershipUtils(entityOwnershipService);
 164         this.interfaceManager = interfaceManager;
 165         this.jobCoordinator = jobCoordinator;
 166         this.l2GatewayCache = l2GatewayCache;
 167         this.elanService = ielanService;
 168         this.dhcpServiceCounters = dhcpServiceCounters;
 169     }
 170
 171     @PostConstruct
 172     public void init() {
 173         initilizeCaches();
 174     }
 175
 176     private void initilizeCaches() {
 177         LOG.trace("Loading designatedDpnsToTunnelIpElanNameCache");
 178         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 179                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 180         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional;
 181         try {
 182             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 183                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 184         } catch (ExecutionException | InterruptedException e) {
 185             LOG.error("initilizeCaches: Exception while reading the DesignatedSwitchesForExternalTunnels DS", e);
 186             return;
 187         }
 188         if (designatedSwitchForTunnelOptional.isPresent()) {
 189             Map<DesignatedSwitchForTunnelKey, DesignatedSwitchForTunnel> keyDesignatedSwitchForTunnelMap =
 190                 designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 191             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : keyDesignatedSwitchForTunnelMap.values()) {
 192                 Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePair =
 193                         designatedDpnsToTunnelIpElanNameCache
 194                                 .get(Uint64.valueOf(designatedSwitchForTunnel.getDpId()));
 195                 if (setOfTunnelIpElanNamePair == null) {
 196                     setOfTunnelIpElanNamePair = new CopyOnWriteArraySet<>();
 197                 }
 198                 Pair<IpAddress, String> tunnelIpElanNamePair =
 199                         new ImmutablePair<>(designatedSwitchForTunnel.getTunnelRemoteIpAddress(),
 200                                 designatedSwitchForTunnel.getElanInstanceName());
 201                 setOfTunnelIpElanNamePair.add(tunnelIpElanNamePair);
 202                 designatedDpnsToTunnelIpElanNameCache.put(Uint64.valueOf(designatedSwitchForTunnel.getDpId()),
 203                         setOfTunnelIpElanNamePair);
 204             }
 205         }
 206         LOG.trace("Loading vniMacAddressToPortCache");
 207         InstanceIdentifier<Ports> inst = InstanceIdentifier.builder(Neutron.class).child(Ports.class).build();
 208         Optional<Ports> optionalPorts;
 209         try {
 210             optionalPorts = SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION,
 211                     inst);
 212         } catch (ExecutionException | InterruptedException e) {
 213             LOG.error("initilizeCaches: Exception while reading the Ports DS", e);
 214             return;
 215         }
 216         if (optionalPorts.isPresent()) {
 217             Map<PortKey, Port> portKeyPortMap = optionalPorts.get().nonnullPort();
 218             for (Port port : portKeyPortMap.values()) {
 219                 if (NeutronUtils.isPortVnicTypeNormal(port)) {
 220                     continue;
 221                 }
 222                 String macAddress = port.getMacAddress().getValue();
 223                 Uuid networkId = port.getNetworkId();
 224                 String segmentationId = DhcpServiceUtils.getSegmentationId(networkId, broker);
 225                 if (segmentationId == null) {
 226                     return;
 227                 }
 228                 updateVniMacToPortCache(Uint64.valueOf(new BigInteger(segmentationId)), macAddress, port);
 229             }
 230         }
 231     }
 232
 233     public Uint64 designateDpnId(IpAddress tunnelIp, String elanInstanceName, List<Uint64> dpns) {
 234         Uint64 designatedDpnId = readDesignatedSwitchesForExternalTunnel(tunnelIp, elanInstanceName);
 235         if (designatedDpnId != null && !designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 236             LOG.trace("Dpn {} already designated for tunnelIp - elan : {} - {}", designatedDpnId, tunnelIp,
 237                     elanInstanceName);
 238             return designatedDpnId;
 239         }
 240         return chooseDpn(tunnelIp, elanInstanceName, dpns);
 241     }
 242
 243     public void installDhcpFlowsForVms(final IpAddress tunnelIp, String elanInstanceName, final List<Uint64> dpns,
 244             final Uint64 designatedDpnId, final String vmMacAddress) {
 245         LOG.trace("In installDhcpFlowsForVms ipAddress {}, elanInstanceName {}, dpn {}, vmMacAddress {}", tunnelIp,
 246                 elanInstanceName, designatedDpnId, vmMacAddress);
 247
 248         String tunnelIpDpnKey = getTunnelIpDpnKey(tunnelIp, designatedDpnId);
 249         jobCoordinator.enqueueJob(getJobKey(tunnelIpDpnKey), () -> {
 250             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 251                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 252                 return Collections.singletonList(
 253                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 254                         dpns.remove(designatedDpnId);
 255                         for (Uint64 dpn : dpns) {
 256                             installDhcpDropAction(dpn, vmMacAddress, tx);
 257                         }
 258                         installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 259                     }));
 260             } else {
 261                 LOG.trace("Exiting installDhcpEntries since this cluster node is not the owner for dpn");
 262             }
 263
 264             return Collections.emptyList();
 265         });
 266
 267         updateLocalCache(tunnelIp, elanInstanceName, vmMacAddress);
 268     }
 269
 270     public void installDhcpFlowsForVms(Uint64 designatedDpnId, Set<String> listVmMacAddress,
 271             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 272         for (String vmMacAddress : listVmMacAddress) {
 273             installDhcpEntries(designatedDpnId, vmMacAddress, tx);
 274         }
 275     }
 276
 277     public void unInstallDhcpFlowsForVms(String elanInstanceName, List<Uint64> dpns, String vmMacAddress) {
 278         unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 279         removeFromLocalCache(elanInstanceName, vmMacAddress);
 280     }
 281
 282     public void unInstallDhcpFlowsForVms(String elanInstanceName, IpAddress tunnelIp, List<Uint64> dpns) {
 283         Pair<IpAddress, String> tunnelIpElanNamePair = new ImmutablePair<>(tunnelIp, elanInstanceName);
 284         Set<String> vmMacs = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 285         LOG.trace("In unInstallFlowsForVms elanInstanceName {}, tunnelIp {}, dpns {}, vmMacs {}",
 286                 elanInstanceName, tunnelIp, dpns, vmMacs);
 287         if (vmMacs == null) {
 288             return;
 289         }
 290         for (String vmMacAddress : vmMacs) {
 291             unInstallDhcpEntriesOnDpns(dpns, vmMacAddress);
 292         }
 293         tunnelIpElanNameToVmMacCache.remove(tunnelIpElanNamePair);
 294     }
 295
 296     @NonNull
 297     public Uint64 readDesignatedSwitchesForExternalTunnel(IpAddress tunnelIp, String elanInstanceName) {
 298         if (tunnelIp == null || elanInstanceName == null || elanInstanceName.isEmpty()) {
 299             return Uint64.ZERO;
 300         }
 301         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 302                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 303                         .child(DesignatedSwitchForTunnel.class,
 304                                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp)).build();
 305         Optional<DesignatedSwitchForTunnel> designatedSwitchForTunnelOptional;
 306         try {
 307             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 308                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 309         } catch (ExecutionException | InterruptedException e) {
 310             LOG.error("readDesignatedSwitchesForExternalTunnel: Exception while reading the DesignatedSwitchForTunnel "
 311                     + "DS for the elan-instance {} tunnelIp {}", elanInstanceName, tunnelIp, e);
 312             return Uint64.ZERO;
 313         }
 314         if (designatedSwitchForTunnelOptional.isPresent()) {
 315             return Uint64.valueOf(designatedSwitchForTunnelOptional.get().getDpId());
 316         }
 317         return Uint64.ZERO;
 318     }
 319
 320     public void writeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 321                                                        String elanInstanceName) {
 322         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 323                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 324         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 325                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 326                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 327         DesignatedSwitchForTunnel designatedSwitchForTunnel =
 328                 new DesignatedSwitchForTunnelBuilder().setDpId(dpnId.longValue())
 329                         .setElanInstanceName(elanInstanceName).setTunnelRemoteIpAddress(tunnelIp)
 330                         .withKey(designatedSwitchForTunnelKey).build();
 331         LOG.trace("Writing into CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp, elanInstanceName,
 332                 dpnId);
 333         MDSALUtil.syncUpdate(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier, designatedSwitchForTunnel);
 334         updateLocalCache(dpnId, tunnelIp, elanInstanceName);
 335     }
 336
 337     public void removeDesignatedSwitchForExternalTunnel(Uint64 dpnId, IpAddress tunnelIp,
 338                                                         String elanInstanceName) {
 339         DesignatedSwitchForTunnelKey designatedSwitchForTunnelKey =
 340                 new DesignatedSwitchForTunnelKey(elanInstanceName, tunnelIp);
 341         InstanceIdentifier<DesignatedSwitchForTunnel> instanceIdentifier =
 342                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class)
 343                         .child(DesignatedSwitchForTunnel.class, designatedSwitchForTunnelKey).build();
 344         LOG.trace("Removing from CONFIG DS tunnelIp {}, elanInstanceName {}, dpnId {}", tunnelIp,
 345                 elanInstanceName, dpnId);
 346         MDSALUtil.syncDelete(broker, LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 347         removeFromLocalCache(dpnId, tunnelIp, elanInstanceName);
 348     }
 349
 350     // This method is called whenever new OVS Switch is added.
 351     public void installDhcpDropActionOnDpn(Uint64 dpId) {
 352         // During controller restart we'll get add for designatedDpns as well and we
 353         // need not install drop flows for those dpns
 354         if (designatedDpnsToTunnelIpElanNameCache.get(dpId) != null) {
 355             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 356             return;
 357         }
 358         // Read from DS since the cache may not get loaded completely in restart scenario
 359         if (isDpnDesignatedDpn(dpId)) {
 360             LOG.trace("The dpn {} is designated DPN need not install drop flows", dpId);
 361             return;
 362         }
 363         List<String> vmMacs = getAllVmMacs();
 364         LOG.trace("Installing drop actions to this new DPN {} VMs {}", dpId, vmMacs);
 365         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 366             for (String vmMacAddress : vmMacs) {
 367                 installDhcpDropAction(dpId, vmMacAddress, tx);
 368             }
 369         }), LOG, "Error writing to the datastore");
 370     }
 371
 372     private boolean isDpnDesignatedDpn(Uint64 dpId) {
 373         InstanceIdentifier<DesignatedSwitchesForExternalTunnels> instanceIdentifier =
 374                 InstanceIdentifier.builder(DesignatedSwitchesForExternalTunnels.class).build();
 375         Optional<DesignatedSwitchesForExternalTunnels> designatedSwitchForTunnelOptional;
 376         try {
 377             designatedSwitchForTunnelOptional = SingleTransactionDataBroker.syncReadOptional(broker,
 378                     LogicalDatastoreType.CONFIGURATION, instanceIdentifier);
 379         } catch (ExecutionException | InterruptedException e) {
 380             LOG.error("isDpnDesignatedDpn: Exception while reading the DesignatedSwitchesForExternalTunnels "
 381                     + "DS for the dpId {}", dpId, e);
 382             return false;
 383         }
 384         if (designatedSwitchForTunnelOptional.isPresent()) {
 385             Map<DesignatedSwitchForTunnelKey, DesignatedSwitchForTunnel> keyDesignatedSwitchForTunnelMap =
 386                     designatedSwitchForTunnelOptional.get().nonnullDesignatedSwitchForTunnel();
 387             for (DesignatedSwitchForTunnel designatedSwitchForTunnel : keyDesignatedSwitchForTunnelMap.values()) {
 388                 if (dpId.equals(Uint64.valueOf(designatedSwitchForTunnel.getDpId()))) {
 389                     return true;
 390                 }
 391             }
 392         }
 393         return false;
 394     }
 395
 396     private List<String> getAllVmMacs() {
 397         List<String> vmMacs = new LinkedList<>();
 398         Collection<Set<String>> listOfVmMacs = tunnelIpElanNameToVmMacCache.values();
 399         for (Set<String> list : listOfVmMacs) {
 400             vmMacs.addAll(list);
 401         }
 402         return vmMacs;
 403     }
 404
 405     public void updateLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 406         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 407         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 408         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 409         if (tunnelIpElanNameSet == null) {
 410             tunnelIpElanNameSet = new CopyOnWriteArraySet<>();
 411         }
 412         tunnelIpElanNameSet.add(tunnelIpElanName);
 413         LOG.trace("Updating designatedDpnsToTunnelIpElanNameCache for designatedDpn {} value {}", designatedDpnId,
 414                 tunnelIpElanNameSet);
 415         designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 416     }
 417
 418     public void updateLocalCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 419         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 420         Set<String> setOfExistingVmMacAddress;
 421         setOfExistingVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanName);
 422         if (setOfExistingVmMacAddress == null) {
 423             setOfExistingVmMacAddress = new CopyOnWriteArraySet<>();
 424         }
 425         setOfExistingVmMacAddress.add(vmMacAddress);
 426         LOG.trace("Updating tunnelIpElanNameToVmMacCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 427                 setOfExistingVmMacAddress);
 428         tunnelIpElanNameToVmMacCache.put(tunnelIpElanName, setOfExistingVmMacAddress);
 429         updateExistingVMTunnelIPCache(tunnelIp, elanInstanceName, vmMacAddress);
 430     }
 431
 432     public void updateExistingVMTunnelIPCache(IpAddress tunnelIp, String elanInstanceName, String vmMacAddress) {
 433         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 434         Set<String> listExistingVmMacAddress;
 435         listExistingVmMacAddress = availableVMCache.get(tunnelIpElanName);
 436         if (listExistingVmMacAddress == null) {
 437             listExistingVmMacAddress = new CopyOnWriteArraySet<>();
 438         }
 439         listExistingVmMacAddress.add(vmMacAddress);
 440         LOG.trace("Updating availableVMCache for tunnelIpElanName {} value {}", tunnelIpElanName,
 441                 listExistingVmMacAddress);
 442         availableVMCache.put(tunnelIpElanName, listExistingVmMacAddress);
 443     }
 444
 445     public void handleDesignatedDpnDown(Uint64 dpnId, List<Uint64> listOfDpns) {
 446         LOG.trace("In handleDesignatedDpnDown dpnId {}, listOfDpns {}", dpnId, listOfDpns);
 447         Set<Pair<IpAddress, String>> setOfTunnelIpElanNamePairs = designatedDpnsToTunnelIpElanNameCache.get(dpnId);
 448         if (setOfTunnelIpElanNamePairs == null || setOfTunnelIpElanNamePairs.isEmpty()) {
 449             LOG.trace("No tunnelIpElanName to handle for dpn {}. Returning", dpnId);
 450         } else {
 451             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 452                 if (!dpnId.equals(DhcpMConstants.INVALID_DPID)) {
 453                     List<String> listOfVms = getAllVmMacs();
 454                     for (String vmMacAddress : listOfVms) {
 455                         unInstallDhcpEntries(dpnId, vmMacAddress, tx);
 456                     }
 457                 }
 458                 for (Pair<IpAddress, String> pair : setOfTunnelIpElanNamePairs) {
 459                     updateCacheAndInstallNewFlows(listOfDpns, pair, tx);
 460                 }
 461             }), LOG, "Error writing to datastore");
 462         }
 463     }
 464
 465     public void updateCacheAndInstallNewFlows(List<Uint64> listOfDpns, Pair<IpAddress, String> pair,
 466             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 467         Uint64 newDesignatedDpn = chooseDpn(pair.getLeft(), pair.getRight(), listOfDpns);
 468         if (newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 469             return;
 470         }
 471         Set<String> setOfVmMacs = tunnelIpElanNameToVmMacCache.get(pair);
 472         if (setOfVmMacs != null && !setOfVmMacs.isEmpty()) {
 473             LOG.trace("Updating DHCP flows for VMs {} with new designated DPN {}", setOfVmMacs, newDesignatedDpn);
 474             installDhcpFlowsForVms(newDesignatedDpn, setOfVmMacs, tx);
 475         }
 476         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(pair.getRight());
 477         if (subnetDhcpData.isPresent()) {
 478             configureDhcpArpRequestResponseFlow(newDesignatedDpn, pair.getRight(), true,
 479                     pair.getLeft(), subnetDhcpData.get().getPortFixedip(), subnetDhcpData.get().getPortMacaddress());
 480         }
 481     }
 482
 483     private void changeExistingFlowToDrop(Pair<IpAddress, String> tunnelIpElanNamePair, Uint64 dpnId,
 484                                           TypedReadWriteTransaction<Configuration> tx)
 485             throws ExecutionException, InterruptedException {
 486         Set<String> setOfVmMacAddress = tunnelIpElanNameToVmMacCache.get(tunnelIpElanNamePair);
 487         if (setOfVmMacAddress == null || setOfVmMacAddress.isEmpty()) {
 488             return;
 489         }
 490         for (String vmMacAddress : setOfVmMacAddress) {
 491             installDhcpDropAction(dpnId, vmMacAddress, tx);
 492         }
 493     }
 494
 495     /**
 496      * Choose a dpn among the list of elanDpns such that it has lowest count of being the designated dpn.
 497      * @param tunnelIp The tunnel Ip address
 498      * @param elanInstanceName The elan instance name
 499      * @param dpns The data path nodes
 500      * @return The designated dpn
 501      */
 502     private Uint64 chooseDpn(IpAddress tunnelIp, String elanInstanceName,
 503             List<Uint64> dpns) {
 504         Uint64 designatedDpnId = DhcpMConstants.INVALID_DPID;
 505         if (dpns != null && dpns.size() != 0) {
 506             List<Uint64> candidateDpns = DhcpServiceUtils.getDpnsForElan(elanInstanceName, broker);
 507             candidateDpns.retainAll(dpns);
 508             LOG.trace("Choosing new dpn for tunnelIp {}, elanInstanceName {}, among elanDpns {}",
 509                     tunnelIp, elanInstanceName, candidateDpns);
 510             boolean elanDpnAvailableFlag = true;
 511             if (candidateDpns.isEmpty()) {
 512                 candidateDpns = dpns;
 513                 elanDpnAvailableFlag = false;
 514             }
 515             int size = 0;
 516             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 517             if (device == null) {
 518                 LOG.trace("Could not find any device for elanInstanceName {} and tunnelIp {}",
 519                         elanInstanceName, tunnelIp);
 520                 handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 521                 return designatedDpnId;
 522             }
 523             for (Uint64 dpn : candidateDpns) {
 524                 String hwvtepNodeId = device.getHwvtepNodeId();
 525                 if (!elanDpnAvailableFlag) {
 526                     if (!isTunnelConfigured(dpn, hwvtepNodeId)) {
 527                         LOG.trace("Tunnel is not configured on dpn {} to TOR {}", dpn, hwvtepNodeId);
 528                         continue;
 529                     }
 530                 } else if (!isTunnelUp(hwvtepNodeId, dpn)) {
 531                     LOG.trace("Tunnel is not up between dpn {} and TOR {}", dpn, hwvtepNodeId);
 532                     continue;
 533                 }
 534                 Set<Pair<IpAddress, String>> tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(dpn);
 535                 if (tunnelIpElanNameSet == null) {
 536                     designatedDpnId = dpn;
 537                     break;
 538                 }
 539                 if (size == 0 || tunnelIpElanNameSet.size() < size) {
 540                     size = tunnelIpElanNameSet.size();
 541                     designatedDpnId = dpn;
 542                 }
 543             }
 544             writeDesignatedSwitchForExternalTunnel(designatedDpnId, tunnelIp, elanInstanceName);
 545             return designatedDpnId;
 546         }
 547         handleUnableToDesignateDpn(tunnelIp, elanInstanceName);
 548         return designatedDpnId;
 549     }
 550
 551     private void handleUnableToDesignateDpn(IpAddress tunnelIp, String elanInstanceName) {
 552         writeDesignatedSwitchForExternalTunnel(DhcpMConstants.INVALID_DPID, tunnelIp, elanInstanceName);
 553     }
 554
 555     private void installDhcpEntries(Uint64 dpnId, String vmMacAddress,
 556             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 557         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 558                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 559     }
 560
 561     public void addOrRemoveDhcpArpFlowforElan(String elanInstanceName, boolean addFlow, String dhcpIpAddress,
 562                                               String dhcpMacAddress) {
 563         LOG.trace("Configure DHCP SR-IOV Arp flows for Elan {} dpns .", elanInstanceName);
 564         for (Entry<Uint64, Set<Pair<IpAddress,String>>> entry : designatedDpnsToTunnelIpElanNameCache.entrySet()) {
 565             Uint64 dpn = entry.getKey();
 566             Set<Pair<IpAddress,String>> tunnelIpElanNameSet = entry.getValue();
 567             for (Pair<IpAddress, String> pair : tunnelIpElanNameSet) {
 568                 if (pair.getRight().equalsIgnoreCase(elanInstanceName)) {
 569                     if (addFlow) {
 570                         LOG.trace("Adding SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 571                                 elanInstanceName, pair.getLeft());
 572                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, true,
 573                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 574                     } else {
 575                         LOG.trace("Deleting SR-IOV DHCP Arp Flows for Elan {} and tunnelIp {}",
 576                                 elanInstanceName, pair.getLeft());
 577                         configureDhcpArpRequestResponseFlow(dpn, elanInstanceName, false,
 578                                 pair.getLeft(), dhcpIpAddress, dhcpMacAddress);
 579                     }
 580                 }
 581             }
 582         }
 583     }
 584
 585
 586     public void configureDhcpArpRequestResponseFlow(Uint64 dpnId, String elanInstanceName, boolean addFlow,
 587                                             IpAddress tunnelIp, String dhcpIpAddress, String dhcpMacAddress) {
 588         L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 589         if (device == null) {
 590             LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 591                     elanInstanceName);
 592         }
 593         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 594             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 595                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 596                 String tunnelInterfaceName = getExternalTunnelInterfaceName(dpnId.toString(),
 597                         device.getHwvtepNodeId());
 598                 int lportTag = interfaceManager.getInterfaceInfo(tunnelInterfaceName).getInterfaceTag();
 599                 InstanceIdentifier<ElanInstance> elanIdentifier = InstanceIdentifier.builder(ElanInstances.class)
 600                         .child(ElanInstance.class, new ElanInstanceKey(elanInstanceName)).build();
 601                 Optional<ElanInstance> optElan = MDSALUtil.read(broker,
 602                         LogicalDatastoreType.CONFIGURATION, elanIdentifier);
 603                 if (optElan.isPresent()) {
 604                     LOG.trace("Configuring the SR-IOV Arp request/response flows for LPort {} ElanTag {}.",
 605                             lportTag, optElan.get().getElanTag());
 606                     Uuid nwUuid = new Uuid(elanInstanceName);
 607                     String strVni = DhcpServiceUtils.getSegmentationId(nwUuid, broker);
 608                     Uint64 vni = strVni != null ? Uint64.valueOf(strVni) : Uint64.valueOf(0);
 609                     if (!vni.equals(Uint64.ZERO)) {
 610                         return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(
 611                             Datastore.CONFIGURATION, tx -> {
 612                                 if (addFlow) {
 613                                     LOG.trace("Installing the SR-IOV DHCP Arp flow for DPN {} Port Ip {}, Lport {}.",
 614                                         dpnId, dhcpIpAddress, lportTag);
 615                                     installDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag,
 616                                         optElan.get().getElanTag().toJava());
 617                                     installDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, elanInstanceName,
 618                                         dhcpIpAddress, dhcpMacAddress);
 619                                 } else {
 620                                     LOG.trace("Uninstalling the SR-IOV DHCP Arp flows for DPN {} Port Ip {}, Lport {}.",
 621                                         dpnId, dhcpIpAddress, lportTag);
 622                                     uninstallDhcpArpRequestFlows(tx, dpnId, vni, dhcpIpAddress, lportTag);
 623                                     uninstallDhcpArpResponderFlows(dpnId, tunnelInterfaceName, lportTag, dhcpIpAddress);
 624                                 }
 625                             }));
 626                     }
 627                 }
 628             }
 629             return Collections.emptyList();
 630         });
 631     }
 632
 633     public  java.util.Optional<SubnetToDhcpPort> getSubnetDhcpPortData(String elanInstanceName) {
 634         java.util.Optional<SubnetToDhcpPort> optSubnetDhcp = java.util.Optional.empty();
 635         Uuid nwUuid = new Uuid(elanInstanceName);
 636         List<Uuid> subnets = DhcpServiceUtils.getSubnetIdsFromNetworkId(broker, nwUuid);
 637         for (Uuid subnet : subnets) {
 638             if (DhcpServiceUtils.isIpv4Subnet(broker, subnet)) {
 639                 optSubnetDhcp = DhcpServiceUtils.getSubnetDhcpPortData(broker, subnet.getValue());
 640                 return optSubnetDhcp;
 641             }
 642         }
 643         return optSubnetDhcp;
 644     }
 645
 646     private void installDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 647                                             Uint64 vni, String dhcpIpAddress, int lportTag, Long elanTag)
 648             throws ExecutionException, InterruptedException {
 649         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 650                 lportTag, elanTag, true, mdsalUtil, tx);
 651     }
 652
 653     private void installDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 654                                               String elanInstanceName, String dhcpIpAddress, String dhcpMacAddress) {
 655         LOG.trace("Adding SR-IOV DHCP ArpResponder for elan {} Lport {} Port Ip {}.",
 656                 elanInstanceName, lportTag, dhcpIpAddress);
 657         ArpResponderInput.ArpReponderInputBuilder builder = new ArpResponderInput.ArpReponderInputBuilder();
 658         builder.setDpId(dpnId.toJava()).setInterfaceName(interfaceName).setSpa(dhcpIpAddress).setSha(dhcpMacAddress)
 659                 .setLportTag(lportTag);
 660         builder.setInstructions(ArpResponderUtil.getInterfaceInstructions(interfaceManager, interfaceName,
 661                 dhcpIpAddress, dhcpMacAddress, itmRpcService));
 662         elanService.addExternalTunnelArpResponderFlow(builder.buildForInstallFlow(), elanInstanceName);
 663     }
 664
 665     private void uninstallDhcpArpResponderFlows(Uint64 dpnId, String interfaceName, int lportTag,
 666                                                 String dhcpIpAddress) {
 667         LOG.trace("Removing SR-IOV DHCP ArpResponder flow for interface {} on DPN {}", interfaceName, dpnId);
 668         ArpResponderInput arpInput = new ArpResponderInput.ArpReponderInputBuilder().setDpId(dpnId.toJava())
 669                 .setInterfaceName(interfaceName).setSpa(dhcpIpAddress)
 670                 .setLportTag(lportTag).buildForRemoveFlow();
 671         elanService.removeArpResponderFlow(arpInput);
 672     }
 673
 674     private void uninstallDhcpArpRequestFlows(TypedReadWriteTransaction<Configuration> tx, Uint64 dpnId,
 675                                               Uint64 vni, String dhcpIpAddress, int lportTag)
 676             throws ExecutionException, InterruptedException {
 677         DhcpServiceUtils.setupDhcpArpRequest(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE, vni, dhcpIpAddress,
 678                 lportTag, null, false, mdsalUtil, tx);
 679     }
 680
 681
 682     public void unInstallDhcpEntries(Uint64 dpnId, String vmMacAddress,
 683             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 684         DhcpServiceUtils.setupDhcpFlowEntry(dpnId, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 685                 vmMacAddress, NwConstants.DEL_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 686     }
 687
 688     private void installDhcpDropAction(Uint64 dpn, String vmMacAddress,
 689             TypedReadWriteTransaction<Configuration> tx) throws ExecutionException, InterruptedException {
 690         DhcpServiceUtils.setupDhcpDropAction(dpn, NwConstants.DHCP_TABLE_EXTERNAL_TUNNEL,
 691                 vmMacAddress, NwConstants.ADD_FLOW, mdsalUtil, dhcpServiceCounters, tx);
 692     }
 693
 694     public List<ListenableFuture<Void>> handleTunnelStateDown(IpAddress tunnelIp, Uint64 interfaceDpn) {
 695         LOG.trace("In handleTunnelStateDown tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 696         if (interfaceDpn == null) {
 697             return Collections.emptyList();
 698         }
 699         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 700             Set<Pair<IpAddress, String>> tunnelElanPairSet =
 701                     designatedDpnsToTunnelIpElanNameCache.get(interfaceDpn);
 702             if (tunnelElanPairSet == null || tunnelElanPairSet.isEmpty()) {
 703                 return Collections.emptyList();
 704             }
 705             for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 706                 IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 707                 if (tunnelIpInDpn.equals(tunnelIp)) {
 708                     if (!checkL2GatewayConnection(tunnelElanPair)) {
 709                         LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 710                                 tunnelElanPair);
 711                         return Collections.emptyList();
 712                     }
 713                 }
 714             }
 715             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 716                 for (Pair<IpAddress, String> tunnelElanPair : tunnelElanPairSet) {
 717                     IpAddress tunnelIpInDpn = tunnelElanPair.getLeft();
 718                     String elanInstanceName = tunnelElanPair.getRight();
 719                     if (tunnelIpInDpn.equals(tunnelIp)) {
 720                         if (!checkL2GatewayConnection(tunnelElanPair)) {
 721                             LOG.trace("Couldn't find device for given tunnelIpElanPair {} in L2GwConnCache",
 722                                     tunnelElanPair);
 723                         }
 724                         List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 725                         dpns.remove(interfaceDpn);
 726                         changeExistingFlowToDrop(tunnelElanPair, interfaceDpn, tx);
 727                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 728                         if (subnetDhcpData.isPresent()) {
 729                             configureDhcpArpRequestResponseFlow(interfaceDpn, elanInstanceName, false,
 730                                     tunnelIpInDpn, subnetDhcpData.get().getPortFixedip(),
 731                                     subnetDhcpData.get().getPortMacaddress());
 732                         }
 733                         updateCacheAndInstallNewFlows(dpns, tunnelElanPair, tx);
 734                     }
 735                 }
 736             }));
 737         }
 738     }
 739
 740     private boolean checkL2GatewayConnection(Pair<IpAddress, String> tunnelElanPair) {
 741         for (L2GatewayDevice device : ElanL2GwCacheUtils.getInvolvedL2GwDevices(tunnelElanPair.getRight())) {
 742             if (Objects.equals(device.getTunnelIp(), tunnelElanPair.getLeft())) {
 743                 return true;
 744             }
 745         }
 746         return false;
 747     }
 748
 749     private String getTunnelIpDpnKey(IpAddress tunnelIp, Uint64 interfaceDpn) {
 750         return tunnelIp.toString() + interfaceDpn;
 751     }
 752
 753     private void removeFromLocalCache(String elanInstanceName, String vmMacAddress) {
 754         for (Entry<Pair<IpAddress, String>, Set<String>> entry : tunnelIpElanNameToVmMacCache.entrySet()) {
 755             Pair<IpAddress, String> pair = entry.getKey();
 756             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
 757                 Set<String> setOfExistingVmMacAddress = entry.getValue();
 758                 if (setOfExistingVmMacAddress == null || setOfExistingVmMacAddress.isEmpty()) {
 759                     continue;
 760                 }
 761                 LOG.trace("Removing vmMacAddress {} from listOfMacs {} for elanInstanceName {}", vmMacAddress,
 762                         setOfExistingVmMacAddress, elanInstanceName);
 763                 setOfExistingVmMacAddress.remove(vmMacAddress);
 764                 if (setOfExistingVmMacAddress.size() > 0) {
 765                     tunnelIpElanNameToVmMacCache.put(pair, setOfExistingVmMacAddress);
 766                     return;
 767                 }
 768                 tunnelIpElanNameToVmMacCache.remove(pair);
 769             }
 770         }
 771     }
 772
 773     public void removeFromLocalCache(Uint64 designatedDpnId, IpAddress tunnelIp, String elanInstanceName) {
 774         Pair<IpAddress, String> tunnelIpElanName = new ImmutablePair<>(tunnelIp, elanInstanceName);
 775         Set<Pair<IpAddress, String>> tunnelIpElanNameSet;
 776         tunnelIpElanNameSet = designatedDpnsToTunnelIpElanNameCache.get(designatedDpnId);
 777         if (tunnelIpElanNameSet != null) {
 778             LOG.trace("Removing tunnelIpElan {} from designatedDpnsToTunnelIpElanNameCache. Existing list {} for "
 779                             + "designatedDpnId {}",
 780                     tunnelIpElanName, tunnelIpElanNameSet, designatedDpnId);
 781             tunnelIpElanNameSet.remove(tunnelIpElanName);
 782             if (tunnelIpElanNameSet.size() != 0) {
 783                 designatedDpnsToTunnelIpElanNameCache.put(designatedDpnId, tunnelIpElanNameSet);
 784             } else {
 785                 designatedDpnsToTunnelIpElanNameCache.remove(designatedDpnId);
 786             }
 787         }
 788     }
 789
 790     public void updateVniMacToPortCache(Uint64 vni, String macAddress, Port port) {
 791         if (macAddress == null) {
 792             return;
 793         }
 794         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 795                 vni, macAddress.toUpperCase(Locale.getDefault()));
 796         LOG.trace("Updating vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}", vni,
 797                 macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair, port);
 798         vniMacAddressToPortCache.put(vniMacAddressPair, port);
 799     }
 800
 801     public void removeVniMacToPortCache(Uint64 vni, String macAddress) {
 802         if (macAddress == null) {
 803             return;
 804         }
 805         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 806                 vni, macAddress.toUpperCase(Locale.getDefault()));
 807         vniMacAddressToPortCache.remove(vniMacAddressPair);
 808     }
 809
 810     @Nullable
 811     public Port readVniMacToPortCache(Uint64 vni, String macAddress) {
 812         if (macAddress == null) {
 813             return null;
 814         }
 815         Pair<Uint64, String> vniMacAddressPair = new ImmutablePair<>(
 816                 vni, macAddress.toUpperCase(Locale.getDefault()));
 817         LOG.trace("Reading vniMacAddressToPortCache with vni {} , mac {} , pair {} and port {}",
 818                 vni, macAddress.toUpperCase(Locale.getDefault()), vniMacAddressPair,
 819                 vniMacAddressToPortCache.get(vniMacAddressPair));
 820         return vniMacAddressToPortCache.get(vniMacAddressPair);
 821     }
 822
 823     public String getExternalTunnelInterfaceName(String sourceNode, String dstNode) {
 824         String tunnelInterfaceName = null;
 825         Class<? extends TunnelTypeBase> tunType = TunnelTypeVxlan.class;
 826         try {
 827             Future<RpcResult<GetExternalTunnelInterfaceNameOutput>> output = itmRpcService
 828                     .getExternalTunnelInterfaceName(new GetExternalTunnelInterfaceNameInputBuilder()
 829                             .setSourceNode(sourceNode).setDestinationNode(dstNode).setTunnelType(tunType).build());
 830
 831             RpcResult<GetExternalTunnelInterfaceNameOutput> rpcResult = output.get();
 832             if (rpcResult.isSuccessful()) {
 833                 tunnelInterfaceName = rpcResult.getResult().getInterfaceName();
 834                 LOG.trace("Tunnel interface name: {}", tunnelInterfaceName);
 835             } else {
 836                 LOG.warn("RPC call to ITM.GetExternalTunnelInterfaceName failed with error: {}", rpcResult.getErrors());
 837             }
 838         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 839             LOG.error("Failed to get external tunnel interface name for sourceNode: {} and dstNode: {}",
 840                     sourceNode, dstNode, e);
 841         }
 842         return tunnelInterfaceName;
 843     }
 844
 845     public static Optional<Node> getNode(DataBroker dataBroker, String physicalSwitchNodeId) {
 846         InstanceIdentifier<Node> psNodeId = HwvtepSouthboundUtils
 847                 .createInstanceIdentifier(new NodeId(physicalSwitchNodeId));
 848         return MDSALUtil.read(LogicalDatastoreType.CONFIGURATION, psNodeId, dataBroker);
 849     }
 850
 851     @Nullable
 852     public RemoteMcastMacs createRemoteMcastMac(Node dstDevice, String logicalSwitchName, IpAddress internalTunnelIp) {
 853         Set<LocatorSet> locators = new HashSet<>();
 854         TerminationPointKey terminationPointKey = HwvtepSouthboundUtils.getTerminationPointKey(
 855                 internalTunnelIp.getIpv4Address().getValue());
 856         HwvtepPhysicalLocatorRef phyLocRef = new HwvtepPhysicalLocatorRef(
 857                 HwvtepSouthboundUtils.createInstanceIdentifier(dstDevice.getNodeId()).child(TerminationPoint.class,
 858                         terminationPointKey));
 859         locators.add(new LocatorSetBuilder().setLocatorRef(phyLocRef).build());
 860
 861         HwvtepLogicalSwitchRef lsRef = new HwvtepLogicalSwitchRef(HwvtepSouthboundUtils
 862                 .createLogicalSwitchesInstanceIdentifier(dstDevice.getNodeId(), new HwvtepNodeName(logicalSwitchName)));
 863
 864         RemoteMcastMacs remoteMcastMacs = new RemoteMcastMacsBuilder()
 865                 .setMacEntryKey(new MacAddress(UNKNOWN_DMAC))
 866                 .setLogicalSwitchRef(lsRef).build();
 867         InstanceIdentifier<RemoteMcastMacs> iid = HwvtepSouthboundUtils.createRemoteMcastMacsInstanceIdentifier(
 868                 dstDevice.getNodeId(), remoteMcastMacs.key());
 869         ReadTransaction transaction = broker.newReadOnlyTransaction();
 870         try {
 871             //TODO do async mdsal read
 872             remoteMcastMacs = transaction.read(LogicalDatastoreType.CONFIGURATION, iid).get().get();
 873             locators.addAll(remoteMcastMacs.getLocatorSet());
 874             return new RemoteMcastMacsBuilder(remoteMcastMacs).setLocatorSet(new ArrayList<>(locators)).build();
 875         } catch (InterruptedException | ExecutionException e) {
 876             LOG.error("Failed to read the macs {}", iid);
 877         } finally {
 878             transaction.close();
 879         }
 880         return null;
 881     }
 882
 883     private void putRemoteMcastMac(TypedWriteTransaction<Configuration> transaction, String elanName,
 884                                    L2GatewayDevice device, IpAddress internalTunnelIp) {
 885         Optional<Node> optionalNode = getNode(broker, device.getHwvtepNodeId());
 886         if (!optionalNode.isPresent()) {
 887             LOG.trace("could not get device node {} ", device.getHwvtepNodeId());
 888             return;
 889         }
 890         Node dstNode = optionalNode.get();
 891         RemoteMcastMacs macs = createRemoteMcastMac(dstNode, elanName, internalTunnelIp);
 892         HwvtepUtils.addRemoteMcastMac(transaction, dstNode.getNodeId(), macs);
 893     }
 894
 895     public void installRemoteMcastMac(final Uint64 designatedDpnId, final IpAddress tunnelIp,
 896                                       final String elanInstanceName) {
 897         if (designatedDpnId.equals(DhcpMConstants.INVALID_DPID)) {
 898             return;
 899         }
 900
 901         jobCoordinator.enqueueJob(getJobKey(elanInstanceName), () -> {
 902             if (!entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
 903                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
 904                 LOG.info("Installing remote McastMac is not executed for this node.");
 905                 return Collections.emptyList();
 906             }
 907
 908             LOG.info("Installing remote McastMac");
 909             L2GatewayDevice device = getDeviceFromTunnelIp(tunnelIp);
 910             if (device == null) {
 911                 LOG.error("Unable to get L2Device for tunnelIp {} and elanInstanceName {}", tunnelIp,
 912                     elanInstanceName);
 913                 return Collections.emptyList();
 914             }
 915             String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(designatedDpnId),
 916                     device.getHwvtepNodeId());
 917             if (tunnelInterfaceName != null) {
 918                 Interface tunnelInterface =
 919                         interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
 920                 if (tunnelInterface == null) {
 921                     LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
 922                     return Collections.emptyList();
 923                 }
 924                 return Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 925                     tx -> putRemoteMcastMac(tx, elanInstanceName, device,
 926                             tunnelInterface.augmentation(IfTunnel.class).getTunnelSource())));
 927             }
 928             return Collections.emptyList();
 929         });
 930     }
 931
 932     @Nullable
 933     private L2GatewayDevice getDeviceFromTunnelIp(IpAddress tunnelIp) {
 934         Collection<L2GatewayDevice> devices = l2GatewayCache.getAll();
 935         LOG.trace("In getDeviceFromTunnelIp devices {}", devices);
 936         for (L2GatewayDevice device : devices) {
 937             if (tunnelIp.equals(device.getTunnelIp())) {
 938                 return device;
 939             }
 940         }
 941         return null;
 942     }
 943
 944     private boolean isTunnelUp(String nodeName, Uint64 dpn) {
 945         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), nodeName);
 946         if (tunnelInterfaceName == null) {
 947             LOG.trace("Tunnel Interface is not present on node {} with dpn {}", nodeName, dpn);
 948             return false;
 949         }
 950         org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state
 951                 .Interface tunnelInterface =
 952                 DhcpServiceUtils.getInterfaceFromOperationalDS(tunnelInterfaceName, broker);
 953         if (tunnelInterface == null) {
 954             LOG.trace("Interface {} is not present in interface state", tunnelInterfaceName);
 955             return false;
 956         }
 957         return tunnelInterface.getOperStatus() == OperStatus.Up;
 958     }
 959
 960     public List<ListenableFuture<Void>> handleTunnelStateUp(IpAddress tunnelIp, Uint64 interfaceDpn) {
 961         LOG.trace("In handleTunnelStateUp tunnelIp {}, interfaceDpn {}", tunnelIp, interfaceDpn);
 962         synchronized (getTunnelIpDpnKey(tunnelIp, interfaceDpn)) {
 963             Set<Pair<IpAddress, String>> tunnelIpElanPair =
 964                     designatedDpnsToTunnelIpElanNameCache.get(DhcpMConstants.INVALID_DPID);
 965             List<Uint64> dpns = DhcpServiceUtils.getListOfDpns(broker);
 966             if (tunnelIpElanPair == null || tunnelIpElanPair.isEmpty()) {
 967                 LOG.trace("There are no undesignated DPNs");
 968                 return Collections.emptyList();
 969             }
 970             return Collections.singletonList(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 971                 for (Pair<IpAddress, String> pair : tunnelIpElanPair) {
 972                     if (tunnelIp.equals(pair.getLeft())) {
 973                         String elanInstanceName = pair.getRight();
 974                         Uint64 newDesignatedDpn = designateDpnId(tunnelIp, elanInstanceName, dpns);
 975                         if (newDesignatedDpn != null && !newDesignatedDpn.equals(DhcpMConstants.INVALID_DPID)) {
 976                             Set<String> vmMacAddress = tunnelIpElanNameToVmMacCache.get(pair);
 977                             if (vmMacAddress != null && !vmMacAddress.isEmpty()) {
 978                                 LOG.trace("Updating DHCP flow for macAddress {} with newDpn {}",
 979                                         vmMacAddress, newDesignatedDpn);
 980                                 installDhcpFlowsForVms(newDesignatedDpn, vmMacAddress, tx);
 981                             }
 982                         }
 983                         java.util.Optional<SubnetToDhcpPort> subnetDhcpData = getSubnetDhcpPortData(elanInstanceName);
 984                         if (subnetDhcpData.isPresent()) {
 985                             configureDhcpArpRequestResponseFlow(newDesignatedDpn, elanInstanceName,
 986                                     true, tunnelIp, subnetDhcpData.get().getPortFixedip(),
 987                                     subnetDhcpData.get().getPortMacaddress());
 988                         }
 989                     }
 990                 }
 991             }));
 992         }
 993     }
 994
 995     private boolean isTunnelConfigured(Uint64 dpn, String hwVtepNodeId) {
 996         String tunnelInterfaceName = getExternalTunnelInterfaceName(String.valueOf(dpn), hwVtepNodeId);
 997         if (tunnelInterfaceName == null) {
 998             return false;
 999         }
1000         Interface tunnelInterface = interfaceManager.getInterfaceInfoFromConfigDataStore(tunnelInterfaceName);
1001         if (tunnelInterface == null) {
1002             LOG.trace("Tunnel Interface is not present {}", tunnelInterfaceName);
1003             return false;
1004         }
1005         return true;
1006     }
1007
1008     public void removeFromAvailableCache(Pair<IpAddress, String> tunnelIpElanName) {
1009         availableVMCache.remove(tunnelIpElanName);
1010     }
1011
1012     private void unInstallDhcpEntriesOnDpns(final List<Uint64> dpns, final String vmMacAddress) {
1013         jobCoordinator.enqueueJob(getJobKey(vmMacAddress), () -> {
1014             if (entityOwnershipUtils.isEntityOwner(HwvtepSouthboundConstants.ELAN_ENTITY_TYPE,
1015                     HwvtepSouthboundConstants.ELAN_ENTITY_NAME)) {
1016                 return Collections.singletonList(
1017                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
1018                         for (final Uint64 dpn : dpns) {
1019                             unInstallDhcpEntries(dpn, vmMacAddress, tx);
1020                         }
1021                     }));
1022             } else {
1023                 LOG.trace("Exiting unInstallDhcpEntries since this cluster node is not the owner for dpn");
1024             }
1025
1026             return Collections.emptyList();
1027         });
1028     }
1029
1030     @Nullable
1031     public IpAddress getTunnelIpBasedOnElan(String elanInstanceName, String vmMacAddress) {
1032         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan elanInstanceName {}", elanInstanceName);
1033         IpAddress tunnelIp = null;
1034         for (Entry<Pair<IpAddress, String>, Set<String>> entry : availableVMCache.entrySet()) {
1035             Pair<IpAddress, String> pair = entry.getKey();
1036             LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan left {} right {}", pair.getLeft(),
1037                     pair.getRight());
1038             if (pair.getRight().trim().equalsIgnoreCase(elanInstanceName.trim())) {
1039                 Set<String> listExistingVmMacAddress = entry.getValue();
1040                 if (listExistingVmMacAddress != null && !listExistingVmMacAddress.isEmpty()
1041                         && listExistingVmMacAddress.contains(vmMacAddress)) {
1042                     tunnelIp = pair.getLeft();
1043                     break;
1044                 }
1045             }
1046         }
1047         LOG.trace("DhcpExternalTunnelManager getTunnelIpBasedOnElan returned tunnelIP {}", tunnelIp);
1048         return tunnelIp;
1049     }
1050
1051     private String getJobKey(final String jobKeySuffix) {
1052         return DhcpMConstants.DHCP_JOB_KEY_PREFIX + jobKeySuffix;
1053     }
1054 }