elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/l2gw/utils/ElanL2GatewayUtils.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.l2gw.utils;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.collect.Lists;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.HashSet;
  23 import java.util.List;
  24 import java.util.Objects;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentMap;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import java.util.concurrent.ScheduledFuture;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.function.Function;
  33 import java.util.stream.Collectors;
  34 import javax.annotation.Nonnull;
  35 import javax.annotation.PreDestroy;
  36 import javax.inject.Inject;
  37 import javax.inject.Singleton;
  38 import org.apache.commons.lang3.tuple.ImmutablePair;
  39 import org.apache.commons.lang3.tuple.Pair;
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  43 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.utils.SystemPropertyReader;
  48 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundConstants;
  49 import org.opendaylight.genius.utils.hwvtep.HwvtepSouthboundUtils;
  50 import org.opendaylight.genius.utils.hwvtep.HwvtepUtils;
  51 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  52 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  53 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  54 import org.opendaylight.netvirt.elan.cache.ElanInstanceDpnsCache;
  55 import org.opendaylight.netvirt.elan.l2gw.ha.HwvtepHAUtil;
  56 import org.opendaylight.netvirt.elan.l2gw.jobs.DeleteL2GwDeviceMacsFromElanJob;
  57 import org.opendaylight.netvirt.elan.l2gw.jobs.DeleteLogicalSwitchJob;
  58 import org.opendaylight.netvirt.elan.utils.ElanClusterUtils;
  59 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  60 import org.opendaylight.netvirt.elan.utils.ElanDmacUtils;
  61 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  62 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  63 import org.opendaylight.netvirt.elan.utils.Scheduler;
  64 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  65 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.Interfaces;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.Interface;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.InterfaceKey;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.IetfYangUtil;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetDpidFromInterfaceInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetDpidFromInterfaceOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.TransportZones;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.transport.zones.transport.zone.subnets.DeviceVteps;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.AddL2GwDeviceInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.AddL2GwDeviceOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.config.rev150710.ElanConfig;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.elan.instance.ExternalTeps;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712.l2gateway.attributes.Devices;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepGlobalAugmentation;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepLogicalSwitchRef;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepMacTableGenericAttributes;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepNodeName;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorAugmentation;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.HwvtepPhysicalLocatorRef;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.LocalUcastMacs;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.LogicalSwitches;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacsKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteUcastMacs;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.locator.set.attributes.LocatorSet;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.physical.port.attributes.VlanBindings;
 103 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 104 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.network.topology.topology.Node;
 105 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 106 import org.opendaylight.yangtools.yang.common.RpcResult;
 107 import org.slf4j.Logger;
 108 import org.slf4j.LoggerFactory;
 109
 110 /**
 111  * It gathers a set of utility methods that handle ELAN configuration in
 112  * external Devices (where external means "not-CSS". As of now: TORs).
 113  *
 114  * <p>It makes use of HwvtepUtils class located under ovsdb/hwvtepsouthbound
 115  * project for low-level mdsal operations
 116  *
 117  * @author eperefr
 118  */
 119 @Singleton
 120 public class ElanL2GatewayUtils {
 121     private static final Logger LOG = LoggerFactory.getLogger(ElanL2GatewayUtils.class);
 122     private static final int DEFAULT_LOGICAL_SWITCH_DELETE_DELAY_SECS = 20;
 123
 124     private final DataBroker broker;
 125     private final ManagedNewTransactionRunner txRunner;
 126     private final ElanDmacUtils elanDmacUtils;
 127     private final ElanItmUtils elanItmUtils;
 128     private final ElanClusterUtils elanClusterUtils;
 129     private final OdlInterfaceRpcService interfaceManagerRpcService;
 130     private final JobCoordinator jobCoordinator;
 131     private final ElanUtils elanUtils;
 132     private final ElanInstanceCache elanInstanceCache;
 133     private final ElanInstanceDpnsCache elanInstanceDpnsCache;
 134
 135     private final ConcurrentMap<Pair<NodeId, String>, ScheduledFuture> logicalSwitchDeletedTasks
 136             = new ConcurrentHashMap<>();
 137     private final ConcurrentMap<Pair<NodeId, String>, DeleteLogicalSwitchJob> deleteJobs = new ConcurrentHashMap<>();
 138     private final Scheduler scheduler;
 139     private final ElanConfig elanConfig;
 140
 141     @Inject
 142     public ElanL2GatewayUtils(DataBroker broker, ElanDmacUtils elanDmacUtils, ElanItmUtils elanItmUtils,
 143             ElanClusterUtils elanClusterUtils, OdlInterfaceRpcService interfaceManagerRpcService,
 144             JobCoordinator jobCoordinator, ElanUtils elanUtils,
 145             Scheduler scheduler, ElanConfig elanConfig, ElanInstanceCache elanInstanceCache,
 146             ElanInstanceDpnsCache elanInstanceDpnsCache) {
 147         this.broker = broker;
 148         this.txRunner = new ManagedNewTransactionRunnerImpl(broker);
 149         this.elanDmacUtils = elanDmacUtils;
 150         this.elanItmUtils = elanItmUtils;
 151         this.elanClusterUtils = elanClusterUtils;
 152         this.interfaceManagerRpcService = interfaceManagerRpcService;
 153         this.jobCoordinator = jobCoordinator;
 154         this.elanUtils = elanUtils;
 155         this.scheduler = scheduler;
 156         this.elanConfig = elanConfig;
 157         this.elanInstanceCache = elanInstanceCache;
 158         this.elanInstanceDpnsCache = elanInstanceDpnsCache;
 159     }
 160
 161     @PreDestroy
 162     public void close() {
 163     }
 164
 165     public long getLogicalSwitchDeleteDelaySecs() {
 166         return elanConfig.getL2gwLogicalSwitchDelaySecs() != null
 167                 ? elanConfig.getL2gwLogicalSwitchDelaySecs() : DEFAULT_LOGICAL_SWITCH_DELETE_DELAY_SECS;
 168     }
 169
 170     /**
 171      * gets the macs addresses for elan interfaces.
 172      *
 173      * @param lstElanInterfaceNames
 174      *            the lst elan interface names
 175      * @return the list
 176      */
 177     public List<PhysAddress> getElanDpnMacsFromInterfaces(Set<String> lstElanInterfaceNames) {
 178         List<PhysAddress> result = new ArrayList<>();
 179         for (String interfaceName : lstElanInterfaceNames) {
 180             ElanInterfaceMac elanInterfaceMac = ElanUtils.getElanInterfaceMacByInterfaceName(broker, interfaceName);
 181             if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
 182                 for (MacEntry macEntry : elanInterfaceMac.getMacEntry()) {
 183                     result.add(macEntry.getMacAddress());
 184                 }
 185             }
 186         }
 187         return result;
 188     }
 189
 190     /**
 191      * Check if phy locator already exists in remote mcast entry.
 192      *
 193      * @param nodeId
 194      *            the node id
 195      * @param remoteMcastMac
 196      *            the remote mcast mac
 197      * @param expectedPhyLocatorIp
 198      *            the expected phy locator ip
 199      * @return true, if successful
 200      */
 201     public static boolean checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(NodeId nodeId,
 202             RemoteMcastMacs remoteMcastMac, IpAddress expectedPhyLocatorIp) {
 203         if (remoteMcastMac != null) {
 204             HwvtepPhysicalLocatorAugmentation expectedPhyLocatorAug = HwvtepSouthboundUtils
 205                     .createHwvtepPhysicalLocatorAugmentation(expectedPhyLocatorIp);
 206             HwvtepPhysicalLocatorRef expectedPhyLocRef = new HwvtepPhysicalLocatorRef(
 207                     HwvtepSouthboundUtils.createPhysicalLocatorInstanceIdentifier(nodeId, expectedPhyLocatorAug));
 208             if (remoteMcastMac.getLocatorSet() != null) {
 209                 for (LocatorSet locatorSet : remoteMcastMac.getLocatorSet()) {
 210                     if (locatorSet.getLocatorRef().equals(expectedPhyLocRef)) {
 211                         LOG.trace("matched phyLocRef: {}", expectedPhyLocRef);
 212                         return true;
 213                     }
 214                 }
 215             }
 216         }
 217         return false;
 218     }
 219
 220     /**
 221      * Gets the remote mcast mac.
 222      *
 223      * @param nodeId
 224      *            the node id
 225      * @param logicalSwitchName
 226      *            the logical switch name
 227      * @param datastoreType
 228      *            the datastore type
 229      * @return the remote mcast mac
 230      */
 231     public RemoteMcastMacs readRemoteMcastMac(NodeId nodeId, String logicalSwitchName,
 232             LogicalDatastoreType datastoreType) {
 233         InstanceIdentifier<LogicalSwitches> logicalSwitch = HwvtepSouthboundUtils
 234                 .createLogicalSwitchesInstanceIdentifier(nodeId, new HwvtepNodeName(logicalSwitchName));
 235         RemoteMcastMacsKey remoteMcastMacsKey = new RemoteMcastMacsKey(new HwvtepLogicalSwitchRef(logicalSwitch),
 236                 new MacAddress(ElanConstants.UNKNOWN_DMAC));
 237         return HwvtepUtils.getRemoteMcastMac(broker, datastoreType, nodeId, remoteMcastMacsKey);
 238     }
 239
 240     /**
 241      * Removes the given MAC Addresses from all the External Devices belonging
 242      * to the specified ELAN.
 243      *
 244      * @param elanInstance
 245      *            the elan instance
 246      * @param macAddresses
 247      *            the mac addresses
 248      */
 249     public void removeMacsFromElanExternalDevices(ElanInstance elanInstance, List<PhysAddress> macAddresses) {
 250         ConcurrentMap<String, L2GatewayDevice> elanL2GwDevices = ElanL2GwCacheUtils
 251                 .getInvolvedL2GwDevices(elanInstance.getElanInstanceName());
 252         for (L2GatewayDevice l2GatewayDevice : elanL2GwDevices.values()) {
 253             removeRemoteUcastMacsFromExternalDevice(l2GatewayDevice.getHwvtepNodeId(),
 254                     elanInstance.getElanInstanceName(), macAddresses);
 255         }
 256     }
 257
 258     /**
 259      * Removes the given MAC Addresses from the specified External Device.
 260      *
 261      * @param deviceNodeId
 262      *            the device node id
 263      * @param macAddresses
 264      *            the mac addresses
 265      * @return the listenable future
 266      */
 267     private ListenableFuture<Void> removeRemoteUcastMacsFromExternalDevice(String deviceNodeId,
 268             String logicalSwitchName, List<PhysAddress> macAddresses) {
 269         NodeId nodeId = new NodeId(deviceNodeId);
 270
 271         // TODO (eperefr)
 272         List<MacAddress> lstMac = macAddresses.stream().filter(Objects::nonNull).map(
 273             physAddress -> new MacAddress(physAddress.getValue())).collect(Collectors.toList());
 274         return HwvtepUtils.deleteRemoteUcastMacs(broker, nodeId, logicalSwitchName, lstMac);
 275     }
 276
 277     public ElanInstance getElanInstanceForUcastLocalMac(LocalUcastMacs localUcastMac) {
 278         Optional<LogicalSwitches> lsOpc = ElanUtils.read(broker, LogicalDatastoreType.OPERATIONAL,
 279                 (InstanceIdentifier<LogicalSwitches>) localUcastMac.getLogicalSwitchRef().getValue());
 280         if (lsOpc.isPresent()) {
 281             LogicalSwitches ls = lsOpc.get();
 282             // Logical switch name is Elan name
 283             String elanName = getElanFromLogicalSwitch(ls.getHwvtepNodeName().getValue());
 284             return elanInstanceCache.get(elanName).orNull();
 285         }
 286         return null;
 287     }
 288
 289     /**
 290      * Install external device local macs in dpn.
 291      *
 292      * @param dpnId
 293      *            the dpn id
 294      * @param l2gwDeviceNodeId
 295      *            the l2gw device node id
 296      * @param elan
 297      *            the elan
 298      * @param interfaceName
 299      *            the interface name
 300      */
 301     public void installL2gwDeviceMacsInDpn(BigInteger dpnId, NodeId l2gwDeviceNodeId, ElanInstance elan,
 302             String interfaceName) {
 303         L2GatewayDevice l2gwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elan.getElanInstanceName(),
 304                 l2gwDeviceNodeId.getValue());
 305         if (l2gwDevice == null) {
 306             LOG.debug("L2 gw device not found in elan cache for device name {}", l2gwDeviceNodeId.getValue());
 307             return;
 308         }
 309
 310         installDmacFlowsOnDpn(dpnId, l2gwDevice, elan, interfaceName);
 311     }
 312
 313     /**
 314      * Install dmac flows on dpn.
 315      *
 316      * @param dpnId
 317      *            the dpn id
 318      * @param l2gwDevice
 319      *            the l2gw device
 320      * @param elan
 321      *            the elan
 322      * @param interfaceName
 323      *            the interface name
 324      */
 325     public void installDmacFlowsOnDpn(BigInteger dpnId, L2GatewayDevice l2gwDevice, ElanInstance elan,
 326             String interfaceName) {
 327         String elanName = elan.getElanInstanceName();
 328
 329         Collection<LocalUcastMacs> l2gwDeviceLocalMacs = l2gwDevice.getUcastLocalMacs();
 330         if (!l2gwDeviceLocalMacs.isEmpty()) {
 331             for (LocalUcastMacs localUcastMac : l2gwDeviceLocalMacs) {
 332                 elanDmacUtils.installDmacFlowsToExternalRemoteMacInBatch(dpnId, l2gwDevice.getHwvtepNodeId(),
 333                         elan.getElanTag(), ElanUtils.getVxlanSegmentationId(elan),
 334                         localUcastMac.getMacEntryKey().getValue(), elanName, interfaceName);
 335             }
 336             LOG.debug("Installing L2gw device [{}] local macs [size: {}] in dpn [{}] for elan [{}]",
 337                     l2gwDevice.getHwvtepNodeId(), l2gwDeviceLocalMacs.size(), dpnId, elanName);
 338         }
 339     }
 340
 341     /**
 342      * Install elan l2gw devices local macs in dpn.
 343      *
 344      * @param dpnId
 345      *            the dpn id
 346      * @param elan
 347      *            the elan
 348      * @param interfaceName
 349      *            the interface name
 350      */
 351     public void installElanL2gwDevicesLocalMacsInDpn(BigInteger dpnId, ElanInstance elan, String interfaceName) {
 352         ConcurrentMap<String, L2GatewayDevice> elanL2GwDevicesFromCache = ElanL2GwCacheUtils
 353                 .getInvolvedL2GwDevices(elan.getElanInstanceName());
 354         if (elanL2GwDevicesFromCache != null) {
 355             for (L2GatewayDevice l2gwDevice : elanL2GwDevicesFromCache.values()) {
 356                 installDmacFlowsOnDpn(dpnId, l2gwDevice, elan, interfaceName);
 357             }
 358         } else {
 359             LOG.debug("No Elan l2 gateway devices in cache for [{}] ", elan.getElanInstanceName());
 360         }
 361     }
 362
 363     public void installL2GwUcastMacInElan(final ElanInstance elan, final L2GatewayDevice extL2GwDevice,
 364             final String macToBeAdded, final LocalUcastMacs localUcastMacs, String interfaceName) {
 365         final String extDeviceNodeId = extL2GwDevice.getHwvtepNodeId();
 366         final String elanInstanceName = elan.getElanInstanceName();
 367         final Collection<DpnInterfaces> elanDpns = getElanDpns(elanInstanceName);
 368         ConcurrentMap<String, L2GatewayDevice> elanL2GwDevices = ElanL2GwCacheUtils
 369                 .getInvolvedL2GwDevices(elanInstanceName);
 370
 371         // Retrieve all participating DPNs in this Elan. Populate this MAC in
 372         // DMAC table.
 373         // Looping through all DPNs in order to add/remove mac flows in their
 374         // DMAC table
 375         if (elanDpns.size() > 0 || elanL2GwDevices.values().size() > 0) {
 376             String jobKey = elanInstanceName + ":" + macToBeAdded;
 377             IpAddress extL2GwDeviceTepIp = extL2GwDevice.getTunnelIp();
 378             List<PhysAddress> macList = Lists.newArrayList(new PhysAddress(macToBeAdded));
 379
 380             elanClusterUtils.runOnlyInOwnerNode(jobKey, "install l2gw macs in dmac table", () -> {
 381                 if (doesLocalUcastMacExistsInCache(extL2GwDevice, localUcastMacs)) {
 382                     List<ListenableFuture<Void>> futures = new ArrayList<>();
 383                     for (DpnInterfaces elanDpn : elanDpns) {
 384                         futures.addAll(elanDmacUtils.installDmacFlowsToExternalRemoteMacInBatch(elanDpn.getDpId(),
 385                                 extDeviceNodeId, elan.getElanTag(), ElanUtils.getVxlanSegmentationId(elan),
 386                                 macToBeAdded, elanInstanceName, interfaceName));
 387                     }
 388                     for (L2GatewayDevice otherDevice : elanL2GwDevices.values()) {
 389                         if (!otherDevice.getHwvtepNodeId().equals(extDeviceNodeId)
 390                                 && !areMLAGDevices(extL2GwDevice, otherDevice)) {
 391                             final String hwvtepId = otherDevice.getHwvtepNodeId();
 392                             final String logicalSwitchName = elanInstanceName;
 393                             futures.add(HwvtepUtils.installUcastMacs(
 394                                     broker, hwvtepId, macList, logicalSwitchName, extL2GwDeviceTepIp));
 395                         }
 396                     }
 397                     return futures;
 398                 } else {
 399                     LOG.trace("Skipping install of dmac flows for mac {} as it is not found in cache",
 400                             macToBeAdded);
 401                 }
 402                 return Collections.emptyList();
 403             });
 404         }
 405     }
 406
 407     /**
 408      * Does local ucast mac exists in cache.
 409      *
 410      * @param elanL2GwDevice
 411      *            the elan L2 Gw device
 412      * @param macAddress
 413      *            the mac address to be verified
 414      * @return true, if successful
 415      */
 416     private static boolean doesLocalUcastMacExistsInCache(L2GatewayDevice elanL2GwDevice, LocalUcastMacs macAddress) {
 417         return elanL2GwDevice.containsUcastMac(macAddress);
 418     }
 419
 420     /**
 421      * Uninstall l2gw macs from other l2gw devices in the elanName provided.
 422      * @param elanName - Elan Name for which other l2gw devices will be scanned.
 423      * @param l2GwDevice - l2gwDevice whose macs are required to be cleared from other devices.
 424      * @param macAddresses - Mac address to be cleared.
 425      */
 426     public void unInstallL2GwUcastMacFromL2gwDevices(final String elanName,
 427                                                      final L2GatewayDevice l2GwDevice,
 428                                                      final Collection<MacAddress> macAddresses) {
 429         if (macAddresses == null || macAddresses.isEmpty()) {
 430             return;
 431         }
 432
 433         if (elanName == null) {
 434             return;
 435         }
 436
 437         DeleteL2GwDeviceMacsFromElanJob job = new DeleteL2GwDeviceMacsFromElanJob(elanName, l2GwDevice,
 438                 macAddresses);
 439         elanClusterUtils.runOnlyInOwnerNode(job.getJobKey(), "delete remote ucast macs in l2gw devices", job);
 440     }
 441
 442     /**
 443      * Uninstall l2gw macs from other DPNs in the elan instance provided.
 444      * @param elan - Elan Instance for which other DPNs will be scanned.
 445      * @param l2GwDevice - l2gwDevice whose macs are required to be cleared from other devices.
 446      * @param macAddresses - Mac address to be cleared.
 447      */
 448     public void unInstallL2GwUcastMacFromElanDpns(final ElanInstance elan, final L2GatewayDevice l2GwDevice,
 449                                                   final Collection<MacAddress> macAddresses) {
 450         if (macAddresses == null || macAddresses.isEmpty()) {
 451             return;
 452         }
 453         if (elan == null || elan.getElanInstanceName() == null) {
 454             LOG.error("Could not delete l2gw ucast macs, Failed to find the elan for device {}",
 455                     l2GwDevice.getHwvtepNodeId());
 456             return;
 457         }
 458
 459         final Collection<DpnInterfaces> elanDpns = getElanDpns(elan.getElanInstanceName());
 460
 461         // Retrieve all participating DPNs in this Elan. Populate this MAC in
 462         // DMAC table. Looping through all DPNs in order to add/remove mac flows
 463         // in their DMAC table
 464         List<ListenableFuture<Void>> result = new ArrayList<>();
 465         for (final MacAddress mac : macAddresses) {
 466             elanClusterUtils.runOnlyInOwnerNode(elan.getElanInstanceName() + ":" + mac.getValue(),
 467                     "delete remote ucast macs in elan DPNs", () -> {
 468                     for (DpnInterfaces elanDpn : elanDpns) {
 469                         BigInteger dpnId = elanDpn.getDpId();
 470                         result.addAll(elanDmacUtils.deleteDmacFlowsToExternalMac(elan.getElanTag(), dpnId,
 471                                 l2GwDevice.getHwvtepNodeId(),
 472                                 IetfYangUtil.INSTANCE.canonizeMacAddress(mac).getValue()));
 473                     }
 474                     return result;
 475                 });
 476         }
 477     }
 478
 479     /**
 480      * Delete elan l2 gateway devices ucast local macs from dpn.
 481      *
 482      * @param elanName
 483      *            the elan name
 484      * @param dpnId
 485      *            the dpn id
 486      */
 487     public void deleteElanL2GwDevicesUcastLocalMacsFromDpn(final String elanName, final BigInteger dpnId) {
 488         ConcurrentMap<String, L2GatewayDevice> elanL2GwDevices = ElanL2GwCacheUtils.getInvolvedL2GwDevices(elanName);
 489         if (elanL2GwDevices == null || elanL2GwDevices.isEmpty()) {
 490             LOG.trace("No L2 gateway devices in Elan [{}] cache.", elanName);
 491             return;
 492         }
 493         final ElanInstance elan = elanInstanceCache.get(elanName).orNull();
 494         if (elan == null) {
 495             LOG.error("Could not find Elan by name: {}", elanName);
 496             return;
 497         }
 498         LOG.info("Deleting Elan [{}] L2GatewayDevices UcastLocalMacs from Dpn [{}]", elanName, dpnId);
 499
 500         final Long elanTag = elan.getElanTag();
 501         for (final L2GatewayDevice l2GwDevice : elanL2GwDevices.values()) {
 502             getL2GwDeviceLocalMacsAndRunCallback(elan.getElanInstanceName(), l2GwDevice, (localMacs) -> {
 503                 for (MacAddress mac : localMacs) {
 504                     String jobKey = elanName + ":" + mac.getValue();
 505                     elanClusterUtils.runOnlyInOwnerNode(jobKey,
 506                         () -> elanDmacUtils.deleteDmacFlowsToExternalMac(elanTag, dpnId,
 507                                 l2GwDevice.getHwvtepNodeId(), mac.getValue()));
 508                 }
 509                 return null;
 510             });
 511         }
 512     }
 513
 514     public void getL2GwDeviceLocalMacsAndRunCallback(String elanName, L2GatewayDevice l2gwDevice,
 515                                                      Function<Collection<MacAddress>, Void> function) {
 516         if (l2gwDevice == null) {
 517             return;
 518         }
 519         Set<MacAddress> macs = new HashSet<>();
 520         Collection<LocalUcastMacs> lstUcastLocalMacs = l2gwDevice.getUcastLocalMacs();
 521         if (!lstUcastLocalMacs.isEmpty()) {
 522             macs.addAll(lstUcastLocalMacs.stream().filter(Objects::nonNull)
 523                     .map(mac -> new MacAddress(mac.getMacEntryKey().getValue().toLowerCase()))
 524                     .collect(Collectors.toList()));
 525         }
 526
 527         InstanceIdentifier<Node> nodeIid = HwvtepSouthboundUtils.createInstanceIdentifier(
 528                 new NodeId(l2gwDevice.getHwvtepNodeId()));
 529         Futures.addCallback(broker.newReadOnlyTransaction().read(LogicalDatastoreType.CONFIGURATION, nodeIid),
 530                 new FutureCallback<Optional<Node>>() {
 531                     @Override
 532                     public void onSuccess(Optional<Node> configNode) {
 533                         if (configNode != null && configNode.isPresent()) {
 534                             HwvtepGlobalAugmentation augmentation = configNode.get().augmentation(
 535                                     HwvtepGlobalAugmentation.class);
 536                             if (augmentation != null && augmentation.getLocalUcastMacs() != null) {
 537                                 macs.addAll(augmentation.getLocalUcastMacs().stream()
 538                                         .filter(mac -> getLogicalSwitchName(mac).equals(elanName))
 539                                         .map(HwvtepMacTableGenericAttributes::getMacEntryKey)
 540                                         .collect(Collectors.toSet()));
 541                             }
 542                             function.apply(macs);
 543                         }
 544                     }
 545
 546                     @Override
 547                     public void onFailure(Throwable throwable) {
 548                         LOG.error("Failed to read config topology node {}", nodeIid);
 549                     }
 550                 }, MoreExecutors.directExecutor());
 551     }
 552
 553     private String getLogicalSwitchName(LocalUcastMacs mac) {
 554         return ((InstanceIdentifier<LogicalSwitches>)mac.getLogicalSwitchRef().getValue())
 555                 .firstKeyOf(LogicalSwitches.class).getHwvtepNodeName().getValue();
 556     }
 557
 558     /**
 559      * Delete elan macs from L2 gateway device.<br>
 560      * This includes deleting ELAN mac table entries plus external device
 561      * UcastLocalMacs which are part of the same ELAN.
 562      *
 563      * @param hwvtepNodeId
 564      *            the hwvtepNodeId
 565      * @param elanName
 566      *            the elan name
 567      * @return the listenable future
 568      */
 569     public ListenableFuture<Void> deleteElanMacsFromL2GatewayDevice(String hwvtepNodeId, String elanName) {
 570         String logicalSwitch = getLogicalSwitchFromElan(elanName);
 571
 572         List<MacAddress> lstElanMacs = getRemoteUcastMacs(new NodeId(hwvtepNodeId), logicalSwitch,
 573                 LogicalDatastoreType.CONFIGURATION);
 574         ListenableFuture<Void> future = HwvtepUtils.deleteRemoteUcastMacs(broker, new NodeId(hwvtepNodeId),
 575                 logicalSwitch, lstElanMacs);
 576
 577         Futures.addCallback(future, new FutureCallback<Void>() {
 578             @Override
 579             public void onSuccess(Void noarg) {
 580                 LOG.trace("Successful in batch deletion of elan [{}] macs from l2gw device [{}]", elanName,
 581                         hwvtepNodeId);
 582             }
 583
 584             @Override
 585             public void onFailure(Throwable error) {
 586                 LOG.warn("Failed during batch delete of elan {} macs from l2gw device {}. "
 587                         + "Retrying with sequential deletes.", elanName, hwvtepNodeId, error);
 588                 if (lstElanMacs != null && !lstElanMacs.isEmpty()) {
 589                     for (MacAddress mac : lstElanMacs) {
 590                         HwvtepUtils.deleteRemoteUcastMac(broker, new NodeId(hwvtepNodeId), logicalSwitch, mac);
 591                     }
 592                 }
 593             }
 594         }, MoreExecutors.directExecutor());
 595
 596         if (LOG.isDebugEnabled()) {
 597             List<String> elanMacs = lstElanMacs.stream().map(MacAddress::getValue).collect(Collectors.toList());
 598             LOG.debug("Deleting elan [{}] macs from node [{}]. Deleted macs = {}", elanName, hwvtepNodeId, elanMacs);
 599         }
 600         return future;
 601     }
 602
 603     /**
 604      * Gets the remote ucast macs from hwvtep node filtering based on logical
 605      * switch.
 606      *
 607      * @param hwvtepNodeId
 608      *            the hwvtep node id
 609      * @param logicalSwitch
 610      *            the logical switch
 611      * @param datastoreType
 612      *            the datastore type
 613      * @return the remote ucast macs
 614      */
 615     public List<MacAddress> getRemoteUcastMacs(NodeId hwvtepNodeId, String logicalSwitch,
 616             LogicalDatastoreType datastoreType) {
 617         List<MacAddress> lstMacs = Collections.emptyList();
 618         Node hwvtepNode = HwvtepUtils.getHwVtepNode(broker, datastoreType, hwvtepNodeId);
 619         if (hwvtepNode != null) {
 620             List<RemoteUcastMacs> remoteUcastMacs = hwvtepNode.augmentation(HwvtepGlobalAugmentation.class)
 621                     .getRemoteUcastMacs();
 622             if (remoteUcastMacs != null && !remoteUcastMacs.isEmpty()) {
 623                 // Filtering remoteUcastMacs based on the logical switch and
 624                 // forming a list of MacAddress
 625                 lstMacs = remoteUcastMacs.stream()
 626                         .filter(mac -> logicalSwitch.equals(mac.getLogicalSwitchRef().getValue()
 627                                 .firstKeyOf(LogicalSwitches.class).getHwvtepNodeName().getValue()))
 628                         .map(HwvtepMacTableGenericAttributes::getMacEntryKey).collect(Collectors.toList());
 629             }
 630         }
 631         return lstMacs;
 632     }
 633
 634     /**
 635      * Install ELAN macs in L2 Gateway device.<br>
 636      * This includes installing ELAN mac table entries plus external device
 637      * UcastLocalMacs which are part of the same ELAN.
 638      *
 639      * @param elanName
 640      *            the elan name
 641      * @param l2GatewayDevice
 642      *            the l2 gateway device which has to be configured
 643      * @return the listenable future
 644      */
 645     public ListenableFuture<Void> installElanMacsInL2GatewayDevice(String elanName,
 646             L2GatewayDevice l2GatewayDevice) {
 647         String logicalSwitchName = getLogicalSwitchFromElan(elanName);
 648         NodeId hwVtepNodeId = new NodeId(l2GatewayDevice.getHwvtepNodeId());
 649
 650         List<RemoteUcastMacs> lstL2GatewayDevicesMacs = getOtherDevicesMacs(elanName, l2GatewayDevice, hwVtepNodeId,
 651                 logicalSwitchName);
 652         List<RemoteUcastMacs> lstElanMacTableEntries = getElanMacTableEntriesMacs(elanName,
 653                 hwVtepNodeId, logicalSwitchName);
 654
 655         List<RemoteUcastMacs> lstRemoteUcastMacs = new ArrayList<>(lstL2GatewayDevicesMacs);
 656         lstRemoteUcastMacs.addAll(lstElanMacTableEntries);
 657
 658         ListenableFuture<Void> future = HwvtepUtils.addRemoteUcastMacs(broker, hwVtepNodeId, lstRemoteUcastMacs);
 659
 660         LOG.info("Added RemoteUcastMacs entries [{}] in config DS. NodeID: {}, LogicalSwitch: {}",
 661                 lstRemoteUcastMacs.size(), hwVtepNodeId.getValue(), logicalSwitchName);
 662         return future;
 663     }
 664
 665     /**
 666      * Gets the l2 gateway devices ucast local macs as remote ucast macs.
 667      *
 668      * @param elanName
 669      *            the elan name
 670      * @param l2GatewayDeviceToBeConfigured
 671      *            the l2 gateway device to be configured
 672      * @param hwVtepNodeId
 673      *            the hw vtep node Id to be configured
 674      * @param logicalSwitchName
 675      *            the logical switch name
 676      * @return the l2 gateway devices macs as remote ucast macs
 677      */
 678     public static List<RemoteUcastMacs> getOtherDevicesMacs(String elanName,
 679             L2GatewayDevice l2GatewayDeviceToBeConfigured, NodeId hwVtepNodeId, String logicalSwitchName) {
 680         List<RemoteUcastMacs> lstRemoteUcastMacs = new ArrayList<>();
 681         ConcurrentMap<String, L2GatewayDevice> elanL2GwDevicesFromCache = ElanL2GwCacheUtils
 682                 .getInvolvedL2GwDevices(elanName);
 683
 684         if (elanL2GwDevicesFromCache != null) {
 685             for (L2GatewayDevice otherDevice : elanL2GwDevicesFromCache.values()) {
 686                 if (l2GatewayDeviceToBeConfigured.getHwvtepNodeId().equals(otherDevice.getHwvtepNodeId())) {
 687                     continue;
 688                 }
 689                 if (!areMLAGDevices(l2GatewayDeviceToBeConfigured, otherDevice)) {
 690                     for (LocalUcastMacs localUcastMac : otherDevice.getUcastLocalMacs()) {
 691                         HwvtepPhysicalLocatorAugmentation physLocatorAug = HwvtepSouthboundUtils
 692                                 .createHwvtepPhysicalLocatorAugmentation(otherDevice.getTunnelIp());
 693                         RemoteUcastMacs remoteUcastMac = HwvtepSouthboundUtils.createRemoteUcastMac(hwVtepNodeId,
 694                             IetfYangUtil.INSTANCE.canonizeMacAddress(localUcastMac.getMacEntryKey()).getValue(),
 695                             localUcastMac.getIpaddr(), logicalSwitchName, physLocatorAug);
 696                         lstRemoteUcastMacs.add(remoteUcastMac);
 697                     }
 698                 }
 699             }
 700         }
 701         return lstRemoteUcastMacs;
 702     }
 703
 704     /**
 705      * Are MLAG devices.
 706      *
 707      * @param l2GatewayDevice
 708      *            the l2 gateway device
 709      * @param otherL2GatewayDevice
 710      *            the other l2 gateway device
 711      * @return true, if both the specified l2 gateway devices are part of same
 712      *         MLAG
 713      */
 714     public static boolean areMLAGDevices(L2GatewayDevice l2GatewayDevice, L2GatewayDevice otherL2GatewayDevice) {
 715         // If tunnel IPs are same, then it is considered to be part of same MLAG
 716         return Objects.equals(l2GatewayDevice.getTunnelIp(), otherL2GatewayDevice.getTunnelIp());
 717     }
 718
 719     /**
 720      * Gets the elan mac table entries as remote ucast macs. <br>
 721      * Note: ELAN MAC table only contains internal switches MAC's. It doesn't
 722      * contain external device MAC's.
 723      *
 724      * @param elanName
 725      *            the elan name
 726      * @param hwVtepNodeId
 727      *            the hw vtep node id
 728      * @param logicalSwitchName
 729      *            the logical switch name
 730      * @return the elan mac table entries as remote ucast macs
 731      */
 732     public List<RemoteUcastMacs> getElanMacTableEntriesMacs(String elanName,
 733             NodeId hwVtepNodeId, String logicalSwitchName) {
 734         List<RemoteUcastMacs> lstRemoteUcastMacs = new ArrayList<>();
 735
 736         MacTable macTable = ElanUtils.getElanMacTable(broker, elanName);
 737         if (macTable == null || macTable.getMacEntry() == null || macTable.getMacEntry().isEmpty()) {
 738             LOG.trace("MacTable is empty for elan: {}", elanName);
 739             return lstRemoteUcastMacs;
 740         }
 741
 742         for (MacEntry macEntry : macTable.getMacEntry()) {
 743             BigInteger dpnId = getDpidFromInterface(macEntry.getInterface());
 744             if (dpnId == null) {
 745                 LOG.error("DPN ID not found for interface {}", macEntry.getInterface());
 746                 continue;
 747             }
 748
 749             IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, hwVtepNodeId);
 750             LOG.trace("Dpn Tep IP: {} for dpnId: {} and nodeId: {}", dpnTepIp, dpnId, hwVtepNodeId.getValue());
 751             if (dpnTepIp == null) {
 752                 LOG.error("TEP IP not found for dpnId {} and nodeId {}", dpnId, hwVtepNodeId.getValue());
 753                 continue;
 754             }
 755             HwvtepPhysicalLocatorAugmentation physLocatorAug = HwvtepSouthboundUtils
 756                     .createHwvtepPhysicalLocatorAugmentation(dpnTepIp);
 757             // TODO: Query ARP cache to get IP address corresponding to the
 758             // MAC
 759             RemoteUcastMacs remoteUcastMac = HwvtepSouthboundUtils.createRemoteUcastMac(hwVtepNodeId,
 760                 IetfYangUtil.INSTANCE.canonizePhysAddress(macEntry.getMacAddress()).getValue(), null /*IpAddress*/,
 761                     logicalSwitchName, physLocatorAug);
 762             lstRemoteUcastMacs.add(remoteUcastMac);
 763         }
 764         return lstRemoteUcastMacs;
 765     }
 766
 767     /**
 768      * Gets the dpid from interface.
 769      *
 770      * @param interfaceName
 771      *            the interface name
 772      * @return the dpid from interface
 773      */
 774     public BigInteger getDpidFromInterface(String interfaceName) {
 775         BigInteger dpId = null;
 776         Future<RpcResult<GetDpidFromInterfaceOutput>> output = interfaceManagerRpcService
 777                 .getDpidFromInterface(new GetDpidFromInterfaceInputBuilder().setIntfName(interfaceName).build());
 778         try {
 779             RpcResult<GetDpidFromInterfaceOutput> rpcResult = output.get();
 780             if (rpcResult != null && rpcResult.isSuccessful()) {
 781                 dpId = rpcResult.getResult().getDpid();
 782             }
 783         } catch (InterruptedException | ExecutionException e) {
 784             LOG.error("Failed to get the DPN ID for interface {}", interfaceName, e);
 785         }
 786         return dpId;
 787     }
 788
 789     /**
 790      * Update vlan bindings in l2 gateway device.
 791      *
 792      * @param nodeId
 793      *            the node id
 794      * @param logicalSwitchName
 795      *            the logical switch name
 796      * @param hwVtepDevice
 797      *            the hardware device
 798      * @param defaultVlanId
 799      *            the default vlan id
 800      * @return the listenable future
 801      */
 802     public ListenableFuture<Void> updateVlanBindingsInL2GatewayDevice(NodeId nodeId, String logicalSwitchName,
 803             Devices hwVtepDevice, Integer defaultVlanId) {
 804         if (hwVtepDevice == null || hwVtepDevice.getInterfaces() == null || hwVtepDevice.getInterfaces().isEmpty()) {
 805             String errMsg = "HwVtepDevice is null or interfaces are empty.";
 806             LOG.error(errMsg);
 807             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 808         }
 809
 810         return txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 811             for (org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712
 812                     .l2gateway.attributes.devices.Interfaces deviceInterface : hwVtepDevice.getInterfaces()) {
 813                 //Removed the check for checking terminationPoint present in OP or not
 814                 //for coniguring vlan bindings
 815                 //As we are not any more dependent on it , plugin takes care of this
 816                 // with port reconcilation.
 817                 List<VlanBindings> vlanBindings = new ArrayList<>();
 818                 if (deviceInterface.getSegmentationIds() != null && !deviceInterface.getSegmentationIds().isEmpty()) {
 819                     for (Integer vlanId : deviceInterface.getSegmentationIds()) {
 820                         vlanBindings.add(HwvtepSouthboundUtils.createVlanBinding(nodeId, vlanId, logicalSwitchName));
 821                     }
 822                 } else {
 823                     // Use defaultVlanId (specified in L2GatewayConnection) if Vlan
 824                     // ID not specified at interface level.
 825                     vlanBindings.add(HwvtepSouthboundUtils.createVlanBinding(nodeId, defaultVlanId, logicalSwitchName));
 826                 }
 827                 HwvtepUtils.mergeVlanBindings(tx, nodeId, hwVtepDevice.getDeviceName(),
 828                         deviceInterface.getInterfaceName(), vlanBindings);
 829             }
 830             LOG.info("Updated Hwvtep VlanBindings in config DS. NodeID: {}, LogicalSwitch: {}", nodeId.getValue(),
 831                     logicalSwitchName);
 832         });
 833     }
 834
 835     /**
 836      * Update vlan bindings in l2 gateway device.
 837      *
 838      * @param nodeId
 839      *            the node id
 840      * @param psName
 841      *            the physical switch name
 842      * @param interfaceName
 843      *            the interface in physical switch
 844      * @param vlanBindings
 845      *            the vlan bindings to be configured
 846      * @return the listenable future
 847      */
 848     public ListenableFuture<Void> updateVlanBindingsInL2GatewayDevice(NodeId nodeId, String psName,
 849             String interfaceName, List<VlanBindings> vlanBindings) {
 850         return txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 851             HwvtepUtils.mergeVlanBindings(tx, nodeId, psName, interfaceName, vlanBindings);
 852             LOG.info("Updated Hwvtep VlanBindings in config DS. NodeID: {}", nodeId.getValue());
 853         });
 854     }
 855
 856     /**
 857      * Delete vlan bindings from l2 gateway device.
 858      *
 859      * @param nodeId
 860      *            the node id
 861      * @param hwVtepDevice
 862      *            the hw vtep device
 863      * @param defaultVlanId
 864      *            the default vlan id
 865      * @return the listenable future
 866      */
 867     public ListenableFuture<Void> deleteVlanBindingsFromL2GatewayDevice(NodeId nodeId, Devices hwVtepDevice,
 868             Integer defaultVlanId) {
 869         if (hwVtepDevice == null || hwVtepDevice.getInterfaces() == null || hwVtepDevice.getInterfaces().isEmpty()) {
 870             String errMsg = "HwVtepDevice is null or interfaces are empty.";
 871             LOG.error(errMsg);
 872             return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 873         }
 874         NodeId physicalSwitchNodeId = HwvtepSouthboundUtils.createManagedNodeId(nodeId, hwVtepDevice.getDeviceName());
 875
 876         return txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 877             for (org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.l2gateways.rev150712
 878                     .l2gateway.attributes.devices.Interfaces deviceInterface : hwVtepDevice.getInterfaces()) {
 879                 String phyPortName = deviceInterface.getInterfaceName();
 880                 if (deviceInterface.getSegmentationIds() != null && !deviceInterface.getSegmentationIds().isEmpty()) {
 881                     for (Integer vlanId : deviceInterface.getSegmentationIds()) {
 882                         HwvtepUtils.deleteVlanBinding(tx, physicalSwitchNodeId, phyPortName, vlanId);
 883                     }
 884                 } else {
 885                     // Use defaultVlanId (specified in L2GatewayConnection) if Vlan
 886                     // ID not specified at interface level.
 887                     HwvtepUtils.deleteVlanBinding(tx, physicalSwitchNodeId, phyPortName, defaultVlanId);
 888                 }
 889             }
 890             LOG.info("Deleted Hwvtep VlanBindings from config DS. NodeID: {}, hwVtepDevice: {}, defaultVlanId: {} ",
 891                     nodeId.getValue(), hwVtepDevice, defaultVlanId);
 892         });
 893     }
 894
 895     /**
 896      * Gets the elan name from logical switch name.
 897      *
 898      * @param logicalSwitchName
 899      *            the logical switch name
 900      * @return the elan name from logical switch name
 901      */
 902     public static String getElanFromLogicalSwitch(String logicalSwitchName) {
 903         // Assuming elan name is same as logical switch name
 904         String elanName = logicalSwitchName;
 905         return elanName;
 906     }
 907
 908     /**
 909      * Gets the logical switch name from elan name.
 910      *
 911      * @param elanName
 912      *            the elan name
 913      * @return the logical switch from elan name
 914      */
 915     public static String getLogicalSwitchFromElan(String elanName) {
 916         // Assuming logical switch name is same as elan name
 917         String logicalSwitchName = elanName;
 918         return logicalSwitchName;
 919     }
 920
 921     /**
 922      * Gets the l2 gateway connection job key.
 923      *
 924      * @param logicalSwitchName
 925      *            the logical switch name
 926      * @return the l2 gateway connection job key
 927      */
 928     public static String getL2GatewayConnectionJobKey(String logicalSwitchName) {
 929         return logicalSwitchName;
 930     }
 931
 932     public static InstanceIdentifier<Interface> getInterfaceIdentifier(InterfaceKey interfaceKey) {
 933         InstanceIdentifier.InstanceIdentifierBuilder<Interface> interfaceInstanceIdentifierBuilder = InstanceIdentifier
 934                 .builder(Interfaces.class).child(Interface.class, interfaceKey);
 935         return interfaceInstanceIdentifierBuilder.build();
 936     }
 937
 938     public static Interface getInterfaceFromConfigDS(InterfaceKey interfaceKey, DataBroker dataBroker) {
 939         InstanceIdentifier<Interface> interfaceId = getInterfaceIdentifier(interfaceKey);
 940         try {
 941             return SingleTransactionDataBroker
 942                     .syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, interfaceId).orNull();
 943         } catch (ReadFailedException e) {
 944             // TODO remove this, and propagate ReadFailedException instead of re-throw RuntimeException
 945             LOG.error("getInterfaceFromConfigDS({}) failed", interfaceKey, e);
 946             throw new RuntimeException(e);
 947         }
 948     }
 949
 950     /**
 951      * Delete l2 gateway device ucast local macs from elan.<br>
 952      * Deletes macs from internal ELAN nodes and also on rest of external l2
 953      * gateway devices which are part of the ELAN.
 954      *
 955      * @param l2GatewayDevice
 956      *            the l2 gateway device whose ucast local macs to be deleted
 957      *            from elan
 958      * @param elanName
 959      *            the elan name
 960      */
 961     public void deleteL2GwDeviceUcastLocalMacsFromElan(L2GatewayDevice l2GatewayDevice,
 962             String elanName) {
 963         LOG.info("Deleting L2GatewayDevice [{}] UcastLocalMacs from elan [{}]", l2GatewayDevice.getHwvtepNodeId(),
 964                 elanName);
 965
 966         ElanInstance elan = elanInstanceCache.get(elanName).orNull();
 967         if (elan == null) {
 968             LOG.error("Could not find Elan by name: {}", elanName);
 969             return;
 970         }
 971
 972         Collection<MacAddress> localMacs = getL2GwDeviceLocalMacs(elanName, l2GatewayDevice);
 973         unInstallL2GwUcastMacFromL2gwDevices(elanName, l2GatewayDevice, localMacs);
 974         unInstallL2GwUcastMacFromElanDpns(elan, l2GatewayDevice, localMacs);
 975     }
 976
 977     public static void createItmTunnels(DataBroker dataBroker, ItmRpcService itmRpcService,
 978                                         String hwvtepId, String psName, IpAddress tunnelIp) {
 979         AddL2GwDeviceInputBuilder builder = new AddL2GwDeviceInputBuilder();
 980         builder.setTopologyId(HwvtepSouthboundConstants.HWVTEP_TOPOLOGY_ID.getValue());
 981         builder.setNodeId(HwvtepSouthboundUtils.createManagedNodeId(new NodeId(hwvtepId), psName).getValue());
 982         builder.setIpAddress(tunnelIp);
 983         try {
 984             deleteStaleTunnelsOfHwvtepInITM(dataBroker, itmRpcService, hwvtepId, psName, tunnelIp);
 985             RpcResult<AddL2GwDeviceOutput> rpcResult = itmRpcService.addL2GwDevice(builder.build()).get();
 986             if (rpcResult.isSuccessful()) {
 987                 LOG.info("Created ITM tunnels for {}", hwvtepId);
 988             } else {
 989                 LOG.error("Failed to create ITM Tunnels: {}", rpcResult.getErrors());
 990             }
 991         } catch (InterruptedException | ExecutionException e) {
 992             LOG.error("RPC to create ITM tunnels failed", e);
 993         }
 994     }
 995
 996     private static void deleteStaleTunnelsOfHwvtepInITM(DataBroker dataBroker,
 997                                                         ItmRpcService itmRpcService,
 998                                                         String globalNodeId,
 999                                                         String psName,
1000                                                         IpAddress tunnelIp) {
1001         try {
1002             Optional<TransportZones> tzonesoptional = readTransportZone(dataBroker);
1003             if (!tzonesoptional.isPresent() || tzonesoptional.get().getTransportZone() == null) {
1004                 return;
1005             }
1006             String psNodeId = globalNodeId + HwvtepHAUtil.PHYSICALSWITCH + psName;
1007             tzonesoptional.get().getTransportZone().stream()
1008                 .filter(transportZone -> transportZone.getSubnets() != null)
1009                 .flatMap(transportZone -> transportZone.getSubnets().stream())
1010                 .filter(subnet -> subnet.getDeviceVteps() != null)
1011                 .flatMap(subnet -> subnet.getDeviceVteps().stream())
1012                 .filter(deviceVteps -> Objects.equals(getPsName(deviceVteps), psName)) //get device with same ps name
1013                 .filter(deviceVteps -> !Objects.equals(psNodeId, deviceVteps.getNodeId())
1014                         || !Objects.equals(tunnelIp, deviceVteps.getIpAddress()))//node id or tunnel ip is changed
1015                 .forEach(deviceVteps -> deleteStaleL2gwTep(dataBroker, itmRpcService, deviceVteps));
1016         } catch (ReadFailedException e) {
1017             LOG.error("Failed delete stale tunnels for {}", globalNodeId);
1018         }
1019     }
1020
1021     private static Optional<TransportZones> readTransportZone(DataBroker dataBroker) throws ReadFailedException {
1022         return new SingleTransactionDataBroker(dataBroker).syncReadOptional(LogicalDatastoreType.CONFIGURATION,
1023                 InstanceIdentifier.builder(TransportZones.class).build());
1024     }
1025
1026     private static Optional<ElanInstances> readElanInstances(DataBroker dataBroker) throws ReadFailedException {
1027         return new SingleTransactionDataBroker(dataBroker).syncReadOptional(LogicalDatastoreType.CONFIGURATION,
1028                 InstanceIdentifier.builder(ElanInstances.class).build());
1029     }
1030
1031     private static String getPsName(DeviceVteps deviceVteps) {
1032         return HwvtepHAUtil.getPsName(HwvtepHAUtil.convertToInstanceIdentifier(deviceVteps.getNodeId()));
1033     }
1034
1035     private static void deleteStaleL2gwTep(DataBroker dataBroker,
1036                                            ItmRpcService itmRpcService,
1037                                            DeviceVteps deviceVteps) {
1038         String psName = HwvtepHAUtil.getPsName(HwvtepHAUtil.convertToInstanceIdentifier(deviceVteps.getNodeId()));
1039         String globalNodeId = HwvtepHAUtil.convertToGlobalNodeId(deviceVteps.getNodeId());
1040         try {
1041             LOG.info("Deleting stale tep {} ", deviceVteps);
1042             L2GatewayUtils.deleteItmTunnels(itmRpcService, globalNodeId, psName, deviceVteps.getIpAddress());
1043             Optional<ElanInstances> optionalElan = readElanInstances(dataBroker);
1044             if (!optionalElan.isPresent()) {
1045                 return;
1046             }
1047             JdkFutures.addErrorLogging(
1048                 new ManagedNewTransactionRunnerImpl(dataBroker).callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1049                     tx -> {
1050                         optionalElan.get().getElanInstance().stream()
1051                         .flatMap(elan -> elan.getExternalTeps().stream()
1052                                 .map(externalTep -> ElanL2GatewayMulticastUtils.buildExternalTepPath(
1053                                         elan.getElanInstanceName(), externalTep.getTepIp())))
1054                         .filter(externalTepIid -> Objects.equals(
1055                                 deviceVteps.getIpAddress(), externalTepIid.firstKeyOf(ExternalTeps.class).getTepIp()))
1056                         .peek(externalTepIid -> LOG.info("Deleting stale external tep {}", externalTepIid))
1057                         .forEach(externalTepIid -> tx.delete(externalTepIid));
1058                     }), LOG, "Failed to delete stale external teps {}", deviceVteps);
1059             Thread.sleep(10000);//TODO remove the sleep currently it waits for interfacemgr to finish the cleanup
1060         } catch (ReadFailedException | InterruptedException e) {
1061             LOG.error("Failed to delete stale l2gw tep {}", deviceVteps, e);
1062         }
1063     }
1064
1065     public static String getNodeIdFromDpnId(BigInteger dpnId) {
1066         return MDSALUtil.NODE_PREFIX + MDSALUtil.SEPARATOR + dpnId.toString();
1067     }
1068
1069     public void scheduleAddDpnMacInExtDevices(String elanName, BigInteger dpId,
1070             List<PhysAddress> staticMacAddresses) {
1071         ConcurrentMap<String, L2GatewayDevice> elanDevices = ElanL2GwCacheUtils.getInvolvedL2GwDevices(elanName);
1072         for (final L2GatewayDevice externalDevice : elanDevices.values()) {
1073             scheduleAddDpnMacsInExtDevice(elanName, dpId, staticMacAddresses, externalDevice);
1074         }
1075     }
1076
1077     public void scheduleAddDpnMacsInExtDevice(final String elanName, BigInteger dpId,
1078             final List<PhysAddress> staticMacAddresses, final L2GatewayDevice externalDevice) {
1079         NodeId nodeId = new NodeId(externalDevice.getHwvtepNodeId());
1080         final IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpId, nodeId);
1081         LOG.trace("Dpn Tep IP: {} for dpnId: {} and nodeId: {}", dpnTepIp, dpId, nodeId);
1082         if (dpnTepIp == null) {
1083             LOG.error("could not install dpn mac in l2gw TEP IP not found for dpnId {} and nodeId {}", dpId, nodeId);
1084             return;
1085         }
1086
1087         //TODO: to  be batched in genius
1088         HwvtepUtils.installUcastMacs(broker, externalDevice.getHwvtepNodeId(), staticMacAddresses, elanName, dpnTepIp);
1089     }
1090
1091     public void scheduleDeleteLogicalSwitch(NodeId hwvtepNodeId, String lsName) {
1092         scheduleDeleteLogicalSwitch(hwvtepNodeId, lsName, false);
1093     }
1094
1095     public void scheduleDeleteLogicalSwitch(final NodeId hwvtepNodeId, final String lsName, final boolean clearUcast) {
1096         final Pair<NodeId, String> nodeIdLogicalSwitchNamePair = new ImmutablePair<>(hwvtepNodeId, lsName);
1097         logicalSwitchDeletedTasks.computeIfAbsent(nodeIdLogicalSwitchNamePair,
1098             (key) -> scheduler.getScheduledExecutorService().schedule(() -> {
1099                 DeleteLogicalSwitchJob deleteLsJob = new DeleteLogicalSwitchJob(broker,
1100                         ElanL2GatewayUtils.this, hwvtepNodeId, lsName, clearUcast);
1101                 jobCoordinator.enqueueJob(deleteLsJob.getJobKey(), deleteLsJob,
1102                         SystemPropertyReader.getDataStoreJobCoordinatorMaxRetries());
1103                 deleteJobs.put(nodeIdLogicalSwitchNamePair, deleteLsJob);
1104                 logicalSwitchDeletedTasks.remove(nodeIdLogicalSwitchNamePair);
1105             }, getLogicalSwitchDeleteDelaySecs(), TimeUnit.SECONDS));
1106     }
1107
1108     public void cancelDeleteLogicalSwitch(final NodeId hwvtepNodeId, final String lsName) {
1109         Pair<NodeId, String> nodeIdLogicalSwitchNamePair = new ImmutablePair<>(hwvtepNodeId, lsName);
1110         ScheduledFuture logicalSwitchDeleteTask = logicalSwitchDeletedTasks.remove(nodeIdLogicalSwitchNamePair);
1111         if (logicalSwitchDeleteTask != null) {
1112             LOG.debug("Delete logical switch {} action on node {} cancelled", lsName, hwvtepNodeId);
1113             logicalSwitchDeleteTask.cancel(true);
1114             DeleteLogicalSwitchJob deleteLogicalSwitchJob = deleteJobs.remove(nodeIdLogicalSwitchNamePair);
1115             if (deleteLogicalSwitchJob != null) {
1116                 deleteLogicalSwitchJob.cancel();
1117             }
1118         }
1119     }
1120
1121     @Nonnull
1122     public Collection<DpnInterfaces> getElanDpns(String elanName) {
1123         Collection<DpnInterfaces> dpnInterfaces = elanInstanceDpnsCache.get(elanName);
1124         if (!dpnInterfaces.isEmpty()) {
1125             return dpnInterfaces;
1126         }
1127
1128         return elanUtils.getElanDPNByName(elanName);
1129     }
1130
1131     /**
1132      * Gets the l2 gw device local macs.
1133      * @param elanName
1134      *            name of the elan
1135      * @param l2gwDevice
1136      *            the l2gw device
1137      * @return the l2 gw device local macs
1138      */
1139     public Collection<MacAddress> getL2GwDeviceLocalMacs(String elanName, L2GatewayDevice l2gwDevice) {
1140         if (l2gwDevice == null) {
1141             return Collections.emptyList();
1142         }
1143         Collection<LocalUcastMacs> lstUcastLocalMacs = l2gwDevice.getUcastLocalMacs();
1144         Set<MacAddress> macs = new HashSet<>();
1145         if (!lstUcastLocalMacs.isEmpty()) {
1146             macs.addAll(lstUcastLocalMacs.stream().filter(Objects::nonNull)
1147                     .map(mac -> new MacAddress(mac.getMacEntryKey().getValue().toLowerCase()))
1148                     .collect(Collectors.toList()));
1149         }
1150         Optional<Node> configNode = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION,
1151                 HwvtepSouthboundUtils.createInstanceIdentifier(new NodeId(l2gwDevice.getHwvtepNodeId())));
1152         if (configNode.isPresent()) {
1153             HwvtepGlobalAugmentation augmentation = configNode.get().augmentation(HwvtepGlobalAugmentation.class);
1154             if (augmentation != null && augmentation.getLocalUcastMacs() != null) {
1155                 macs.addAll(augmentation.getLocalUcastMacs().stream()
1156                         .filter(mac -> getLogicalSwitchName(mac).equals(elanName))
1157                         .map(HwvtepMacTableGenericAttributes::getMacEntryKey)
1158                         .collect(Collectors.toSet()));
1159             }
1160         }
1161         return macs;
1162     }
1163 }