elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlan;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Queue;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentLinkedQueue;
  28 import javax.annotation.PostConstruct;
  29 import javax.inject.Inject;
  30 import javax.inject.Singleton;
  31 import org.apache.commons.lang3.StringUtils;
  32 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  33 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  37 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  38 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  40 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.itm.globals.ITMConstants;
  43 import org.opendaylight.genius.mdsalutil.FlowEntity;
  44 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  45 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.mdsalutil.MatchInfo;
  48 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  49 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  50 import org.opendaylight.genius.mdsalutil.NwConstants;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  56 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  57 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  58 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  61 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  62 import org.opendaylight.genius.utils.ServiceIndex;
  63 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  64 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  65 import org.opendaylight.netvirt.elan.ElanException;
  66 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  67 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  68 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  69 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  70 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  71 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  72 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  73 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  74 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  75 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  76 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  77 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  78 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  79 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  81 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  82 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 123 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 124 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 125 import org.slf4j.Logger;
 126 import org.slf4j.LoggerFactory;
 127
 128 /**
 129  * Class in charge of handling creations, modifications and removals of
 130  * ElanInterfaces.
 131  *
 132  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 133  */
 134 @Singleton
 135 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager> {
 136     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 137     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 138     private static final boolean SH_FLAG_SET = true;
 139     private static final boolean SH_FLAG_UNSET = false;
 140
 141     private final DataBroker broker;
 142     private final ManagedNewTransactionRunner txRunner;
 143     private final IMdsalApiManager mdsalManager;
 144     private final IInterfaceManager interfaceManager;
 145     private final IdManagerService idManager;
 146     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 147     private final INeutronVpnManager neutronVpnManager;
 148     private final ElanItmUtils elanItmUtils;
 149     private final ElanEtreeUtils elanEtreeUtils;
 150     private final ElanL2GatewayUtils elanL2GatewayUtils;
 151     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 152     private final ElanUtils elanUtils;
 153     private final JobCoordinator jobCoordinator;
 154     private final ElanInstanceCache elanInstanceCache;
 155     private final ElanInterfaceCache elanInterfaceCache;
 156
 157     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 158         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 159
 160     @Inject
 161     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 162                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 163                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 164                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 165                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 166                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 167                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 168                                 final ElanInstanceCache elanInstanceCache,
 169                                 final ElanInterfaceCache elanInterfaceCache) {
 170         super(ElanInterface.class, ElanInterfaceManager.class);
 171         this.broker = dataBroker;
 172         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 173         this.idManager = managerService;
 174         this.mdsalManager = mdsalApiManager;
 175         this.interfaceManager = interfaceManager;
 176         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 177         this.neutronVpnManager = neutronVpnManager;
 178         this.elanItmUtils = elanItmUtils;
 179         this.elanEtreeUtils = elanEtreeUtils;
 180         this.elanL2GatewayUtils = elanL2GatewayUtils;
 181         this.elanUtils = elanUtils;
 182         this.jobCoordinator = jobCoordinator;
 183         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 184         this.elanInstanceCache = elanInstanceCache;
 185         this.elanInterfaceCache = elanInterfaceCache;
 186     }
 187
 188     @Override
 189     @PostConstruct
 190     public void init() {
 191         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 192     }
 193
 194     @Override
 195     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 196         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 197     }
 198
 199     @Override
 200     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 201         String interfaceName = del.getName();
 202         ElanInstance elanInfo = elanInstanceCache.get(del.getElanInstanceName()).orNull();
 203         /*
 204          * Handling in case the elan instance is deleted.If the Elan instance is
 205          * deleted, there is no need to explicitly delete the elan interfaces
 206          */
 207         if (elanInfo == null) {
 208             return;
 209         }
 210         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 211         if (interfaceInfo == null && elanInfo.isExternal()) {
 212             // In deleting external network, the underlying ietf Inteface might have been removed
 213             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 214             // from Operational DS instead
 215             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 216         }
 217         String elanInstanceName = elanInfo.getElanInstanceName();
 218         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 219                 interfaceName, interfaceInfo, this);
 220         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 221     }
 222
 223     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 224             InterfaceInfo interfaceInfo) {
 225         String elanName = elanInfo.getElanInstanceName();
 226         boolean isLastElanInterface = false;
 227         boolean isLastInterfaceOnDpn = false;
 228         BigInteger dpId = null;
 229         long elanTag = elanInfo.getElanTag();
 230         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 231         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 232         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 233         if (elanState == null) {
 234             interfaceTx.cancel();
 235             return Collections.emptyList();
 236         }
 237         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 238         List<String> elanInterfaces = elanState.getElanInterfaces();
 239         if (elanInterfaces.isEmpty()) {
 240             isLastElanInterface = true;
 241         }
 242         if (interfaceInfo != null) {
 243             dpId = interfaceInfo.getDpId();
 244             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 245                     interfaceName, elanTag, interfaceTx);
 246             /*
 247              * If there are not elan ports, remove the unknown dmac, terminating
 248              * service table flows, remote/local bc group
 249              */
 250             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 251                     || dpnInterfaces.getInterfaces().isEmpty()) {
 252                 // No more Elan Interfaces in this DPN
 253                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 254                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 255                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 256                 }
 257                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 258                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 259                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 260                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 261                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 262                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 263                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 264                         elanUtils.removeTerminatingServiceAction(dpId, elanInfo.getSegmentationId().intValue());
 265                     }
 266                     unsetExternalTunnelTable(dpId, elanInfo);
 267                 }
 268                 isLastInterfaceOnDpn = true;
 269             } else {
 270                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 271             }
 272         }
 273
 274         List<ListenableFuture<Void>> futures = new ArrayList<>();
 275         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 276         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 277
 278         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 279             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 280         }
 281         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 282                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 283         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 284                 ElanConstants.JOB_MAX_RETRIES);
 285
 286         return futures;
 287     }
 288
 289     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 290             WriteTransaction deleteFlowGroupTx) {
 291         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 292         if (etreeLeafTag != null) {
 293             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 294             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 295         }
 296     }
 297
 298     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 299             WriteTransaction deleteFlowGroupTx) {
 300         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 301         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 302     }
 303
 304     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 305             WriteTransaction deleteFlowGroupTx) {
 306         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 307         if (etreeInstance != null) {
 308             BigInteger dpnId = interfaceInfo.getDpId();
 309             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 310             List<Bucket> listBuckets = new ArrayList<>();
 311             int bucketId = 0;
 312             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 313             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 314                     MDSALUtil.buildBucketLists(listBuckets));
 315             LOG.trace("deleted the localBroadCast Group:{}", group);
 316             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 317         }
 318     }
 319
 320     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 321             WriteTransaction deleteFlowGroupTx) {
 322         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 323         if (etreeInstance != null) {
 324             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 325             int bucketId = 0;
 326             int actionKey = 0;
 327             List<Bucket> listBuckets = new ArrayList<>();
 328             List<Action> listAction = new ArrayList<>();
 329             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 330             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 331                     MDSALUtil.WATCH_GROUP));
 332             bucketId++;
 333             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 334             BigInteger dpnId = interfaceInfo.getDpId();
 335             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 336             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 337                     MDSALUtil.buildBucketLists(listBuckets));
 338             LOG.trace("deleting the remoteBroadCast group:{}", group);
 339             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 340         }
 341     }
 342
 343     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 344         String elanName = elanInfo.getElanInstanceName();
 345         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 346         if (elanState == null) {
 347             return elanState;
 348         }
 349         List<String> elanInterfaces = elanState.getElanInterfaces();
 350         boolean isRemoved = elanInterfaces.remove(interfaceName);
 351         if (!isRemoved) {
 352             return elanState;
 353         }
 354
 355         if (elanInterfaces.isEmpty()) {
 356             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 357             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 358             tx.delete(LogicalDatastoreType.OPERATIONAL,
 359                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 360         } else {
 361             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 362                     .setKey(new ElanKey(elanName)).build();
 363             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 364                     updateElanState);
 365         }
 366         return elanState;
 367     }
 368
 369     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 370         // removing the ElanInterface from the config data_store if interface is
 371         // not present in Interface config DS
 372         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 373                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 374             tx.delete(LogicalDatastoreType.CONFIGURATION,
 375                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 376         }
 377     }
 378
 379     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 380             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 381         String elanName = elanInfo.getElanInstanceName();
 382         List<ListenableFuture<Void>> futures = new ArrayList<>();
 383         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(flowTx -> {
 384             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(interfaceTx -> {
 385                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 386                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 387                 Optional<ElanInterfaceMac> existingElanInterfaceMac =
 388                         interfaceTx.read(LogicalDatastoreType.OPERATIONAL, elanInterfaceId).checkedGet();
 389                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 390                 if (interfaceInfo != null) {
 391                     if (existingElanInterfaceMac.isPresent()) {
 392                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 393                         if (existingMacEntries != null) {
 394                             List<PhysAddress> macAddresses = new ArrayList<>();
 395                             for (MacEntry macEntry : existingMacEntries) {
 396                                 PhysAddress macAddress = macEntry.getMacAddress();
 397                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 398                                         macAddress.getValue(), interfaceName);
 399                                 Optional<MacEntry> macEntryOptional =
 400                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 401                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 402                                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 403                                             ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 404                                 }
 405                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 406                                 macAddresses.add(macAddress);
 407                             }
 408
 409                             // Removing all those MACs from External Devices belonging
 410                             // to this ELAN
 411                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 412                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 413                             }
 414                         }
 415                     }
 416                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 417                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 418                 } else if (existingElanInterfaceMac.isPresent()) {
 419                     // Interface does not exist in ConfigDS, so lets remove everything
 420                     // about that interface related to Elan
 421                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 422                     if (macEntries != null) {
 423                         for (MacEntry macEntry : macEntries) {
 424                             PhysAddress macAddress = macEntry.getMacAddress();
 425                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 426                                 interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 427                                         ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 428                             }
 429                         }
 430                     }
 431                 }
 432                 if (existingElanInterfaceMac.isPresent()) {
 433                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 434                 }
 435                 unbindService(interfaceName, interfaceTx);
 436                 deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 437             }));
 438         }));
 439         return futures;
 440     }
 441
 442     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 443                                                                          String interfaceName, long elanTag,
 444                                                                          WriteTransaction tx) {
 445         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 446         if (dpnInterfaces != null) {
 447             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 448             if (interfaceLists != null) {
 449                 interfaceLists.remove(interfaceName);
 450             }
 451
 452             if (interfaceLists == null || interfaceLists.isEmpty()) {
 453                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 454                 deleteElanDpnInterface(elanName, dpId, tx);
 455             } else {
 456                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 457             }
 458         }
 459         return dpnInterfaces;
 460     }
 461
 462     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 463         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 464         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 465             BigInteger currentDpId = dpnInterface.getDpId();
 466             if (!currentDpId.equals(dpId)) {
 467                 for (String elanInterface : dpnInterface.getInterfaces()) {
 468                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 469                     if (macs == null || macs.getMacEntry() == null) {
 470                         continue;
 471                     }
 472                     for (MacEntry mac : macs.getMacEntry()) {
 473                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 474                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 475                     }
 476                 }
 477             }
 478         }
 479     }
 480
 481     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 482         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 483         if (etreeLeafTag != null) {
 484             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 485         }
 486     }
 487
 488     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 489         mdsalManager
 490                 .removeFlow(dpId,
 491                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 492                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 493                                         mac.getMacAddress().getValue(), elanTag)));
 494     }
 495
 496     /*
 497     * Possible Scenarios for update
 498     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 499         then all {1,2,3,4} should be removed
 500
 501         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 502         then all {1,2,3,4} should be added
 503
 504         c. if orig = {1,2,3,4} updated={2,3,4}
 505         then 1 should be removed
 506
 507         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 508         then we should just add 5
 509
 510         e. if orig = {1,2,3,4} updated={2,3,4,5}
 511         then 1 should be removed , 5 should be added
 512     * */
 513     @Override
 514     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 515         // updating the static-Mac Entries for the existing elanInterface
 516         String elanName = update.getElanInstanceName();
 517         String interfaceName = update.getName();
 518
 519         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 520         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 521         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 522         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 523
 524         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 525                 deletedEntry.getMacAddress()));
 526
 527         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 528         * Hence add the macentries for the same.*/
 529         for (StaticMacEntries staticMacEntry : updatedEntries) {
 530             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 531                     staticMacEntry.getMacAddress());
 532             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 533                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 534             WriteTransaction tx = broker.newWriteOnlyTransaction();
 535             if (existingMacEntry.isPresent()) {
 536                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 537                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 538                         tx);
 539             } else {
 540                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 541                         elanName, interfaceName, staticMacEntry, tx);
 542             }
 543             ElanUtils.waitForTransactionToComplete(tx);
 544         }
 545     }
 546
 547     @Override
 548     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 549         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 550             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 551             String interfaceName = elanInterfaceAdded.getName();
 552             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 553             if (interfaceInfo == null) {
 554                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 555                 return;
 556             }
 557             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 558
 559             if (elanInstance == null) {
 560                 elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 561                         .setDescription(elanInterfaceAdded.getDescription()).build();
 562                 // Add the ElanInstance in the Configuration data-store
 563                 List<String> elanInterfaces = new ArrayList<>();
 564                 elanInterfaces.add(interfaceName);
 565                 elanInstance = ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 566             }
 567
 568             Long elanTag = elanInstance.getElanTag();
 569             // If elan tag is not updated, then put the elan interface into
 570             // unprocessed entry map and entry. Let entries
 571             // in this map get processed during ELAN update DCN.
 572             if (elanTag == null) {
 573                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 574                 if (elanInterfaces == null) {
 575                     elanInterfaces = new ConcurrentLinkedQueue<>();
 576                 }
 577                 elanInterfaces.add(elanInterfaceAdded);
 578                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 579                 return;
 580             }
 581             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 582                     interfaceInfo, elanInstance, this);
 583             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 584         }), LOG, "Error procedding added ELAN interface");
 585     }
 586
 587     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 588         List<ListenableFuture<Void>> futures = new ArrayList<>();
 589         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 590         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 591             return futures;
 592         }
 593         for (ElanInterface elanInterface : elanInterfaces) {
 594             String interfaceName = elanInterface.getName();
 595             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 596             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 597         }
 598         return futures;
 599     }
 600
 601     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 602             WriteTransaction writeFlowGroupTx) throws ElanException {
 603         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 604                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 605         List<DpnInterfaces> dpnInterfaceLists = null;
 606         if (elanDpnInterfacesList != null) {
 607             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 608         }
 609         if (dpnInterfaceLists == null) {
 610             dpnInterfaceLists = new ArrayList<>();
 611         }
 612         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 613             BigInteger dstDpId = interfaceInfo.getDpId();
 614             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 615                 continue;
 616             }
 617             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 618             for (String remoteIf : remoteElanInterfaces) {
 619                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 620                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 621                 if (elanIfMac == null) {
 622                     continue;
 623                 }
 624                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 625                 if (remoteMacEntries != null) {
 626                     for (MacEntry macEntry : remoteMacEntries) {
 627                         String macAddress = macEntry.getMacAddress().getValue();
 628                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 629                                 dstDpId, elanInstance.getElanInstanceName());
 630                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 631                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 632                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 633                     }
 634                 }
 635             }
 636         }
 637     }
 638
 639     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 640             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 641         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 642         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 643         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 644
 645         String interfaceName = elanInterface.getName();
 646         String elanInstanceName = elanInterface.getElanInstanceName();
 647
 648         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 649         WriteTransaction tx = broker.newWriteOnlyTransaction();
 650         if (elanInfo == null) {
 651             List<String> elanInterfaces = new ArrayList<>();
 652             elanInterfaces.add(interfaceName);
 653             ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 654         } else {
 655             createElanStateList(elanInstanceName, interfaceName, tx);
 656         }
 657         boolean isFirstInterfaceInDpn = false;
 658         // Specific actions to the DPN where the ElanInterface has been added,
 659         // for example, programming the
 660         // External tunnel table if needed or adding the ElanInterface to the
 661         // DpnInterfaces in the operational DS.
 662         BigInteger dpId = interfaceInfo.getDpId();
 663         DpnInterfaces dpnInterfaces = null;
 664         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 665             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 666                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 667             Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 668                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 669             if (!existingElanDpnInterfaces.isPresent()) {
 670                 isFirstInterfaceInDpn = true;
 671                 // ELAN's 1st ElanInterface added to this DPN
 672                 dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 673                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 674                 // table, but only if Elan has VNI
 675                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 676                     setExternalTunnelTable(dpId, elanInstance);
 677                 }
 678                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 679             } else {
 680                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 681                 elanInterfaces.add(interfaceName);
 682                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 683                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 684                 }
 685                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 686             }
 687         }
 688
 689         // add code to install Local/Remote BC group, unknow DMAC entry,
 690         // terminating service table flow entry
 691         // call bindservice of interfacemanager to create ingress table flow
 692         // enty.
 693         // Add interface to the ElanInterfaceForwardingEntires Container
 694         createElanInterfaceTablesList(interfaceName, tx);
 695         List<ListenableFuture<Void>> futures = new ArrayList<>();
 696         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 697         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces, isFirstInterfaceInDpn);
 698
 699         // add the vlan provider interface to remote BC group for the elan
 700         // for internal vlan networks
 701         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 702             if (interfaceManager.isExternalInterface(interfaceName)) {
 703                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 704                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 705             }
 706         }
 707
 708         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 709             //update the remote-DPNs remoteBC group entry with Tunnels
 710             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 711             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 712         }
 713
 714         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 715         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 716                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 717         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 718         return futures;
 719     }
 720
 721     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 722             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 723             throws ElanException {
 724         String elanInstanceName = elanInstance.getElanInstanceName();
 725         String interfaceName = elanInterface.getName();
 726         WriteTransaction tx = broker.newWriteOnlyTransaction();
 727         BigInteger dpId = interfaceInfo.getDpId();
 728         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 729         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 730                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 731
 732         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 733         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 734
 735         boolean isInterfaceOperational = isOperational(interfaceInfo);
 736         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 737             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 738                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 739                         staticMacEntry.getMacAddress());
 740                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 741                         LogicalDatastoreType.OPERATIONAL, macId);
 742                 if (existingMacEntry.isPresent()) {
 743                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 744                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 745                             existingMacEntry.get(), tx);
 746                 } else {
 747                     elanForwardingEntriesHandler
 748                             .addElanInterfaceForwardingTableList(elanInstanceName, interfaceName, staticMacEntry, tx);
 749                 }
 750
 751                 if (isInterfaceOperational) {
 752                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 753                     // DPNs
 754                     String macAddress = staticMacEntry.getMacAddress().getValue();
 755                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 756                             macAddress, elanInstanceName, interfaceName);
 757                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 758                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 759                 }
 760             }
 761
 762             if (isInterfaceOperational) {
 763                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 764                 // on purpose.
 765                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 766                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 767                 }
 768                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 769                         staticMacAddresses);
 770             }
 771         }
 772         List<ListenableFuture<Void>> futures = new ArrayList<>();
 773         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 774         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 775         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 776             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 777             try {
 778                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 779                 if (neutronPort != null) {
 780                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 781                 }
 782             } catch (IllegalArgumentException ex) {
 783                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 784             }
 785         }
 786         return futures;
 787     }
 788
 789     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 790             PhysAddress physAddress) {
 791         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 792         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 793         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 794                 LogicalDatastoreType.OPERATIONAL, macId);
 795
 796         if (!existingMacEntry.isPresent()) {
 797             return;
 798         }
 799
 800         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 801                 .setKey(new MacEntryKey(physAddress)).build();
 802         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 803                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 804     }
 805
 806     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 807         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 808                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 809     }
 810
 811     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 812             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 813             WriteTransaction writeFlowGroupTx) throws ElanException {
 814         if (!isOperational(interfaceInfo)) {
 815             return;
 816         }
 817         BigInteger dpId = interfaceInfo.getDpId();
 818         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 819             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 820         }
 821         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 822         if (isFirstInterfaceInDpn) {
 823             // Terminating Service , UnknownDMAC Table.
 824             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 825             // for ELAN Instance is VxLAN
 826             if (isVxlan(elanInstance)) {
 827                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 828             }
 829             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 830             /*
 831              * Install remote DMAC flow. This is required since this DPN is
 832              * added later to the elan instance and remote DMACs of other
 833              * interfaces in this elan instance are not present in the current
 834              * dpn.
 835              */
 836             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 837                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 838                         elanInstance.getElanInstanceName());
 839                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 840             }
 841         }
 842         // bind the Elan service to the Interface
 843         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 844     }
 845
 846     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 847             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn) {
 848         if (!isOperational(interfaceInfo)) {
 849             return;
 850         }
 851         // LocalBroadcast Group creation with elan-Interfaces
 852         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 853         if (isFirstInterfaceInDpn) {
 854             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 855             BigInteger dpId = interfaceInfo.getDpId();
 856             // RemoteBroadcast Group creation
 857             try {
 858                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 859             } catch (InterruptedException e1) {
 860                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 861             }
 862             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 863             try {
 864                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 865             } catch (InterruptedException e1) {
 866                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 867             }
 868         }
 869     }
 870
 871     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 872             WriteTransaction writeFlowGroupTx) {
 873         int ifTag = interfaceInfo.getInterfaceTag();
 874         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 875                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 876                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 877                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 878                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 879
 880         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 881
 882         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 883                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 884                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 885                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 886
 887         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 888     }
 889
 890     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 891             WriteTransaction deleteFlowGroupTx) {
 892         int ifTag = interfaceInfo.getInterfaceTag();
 893         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 894                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 895                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 896                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 897                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 898
 899         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 900
 901         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 902                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 903                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 904                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 905
 906         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 907     }
 908
 909     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 910                                                      InterfaceInfo interfaceInfo, long elanTag) {
 911         return elanL2GatewayMulticastUtils.getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(),
 912                 bucketKeyStart, elanTag);
 913     }
 914
 915     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 916         int elanTag = elanInfo.getElanTag().intValue();
 917         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 918         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 919         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 920         if (etreeInstance != null) {
 921             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 922             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 923             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 924         }
 925     }
 926
 927     @SuppressWarnings("checkstyle:IllegalCatch")
 928     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
 929         int bucketId = 0;
 930         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 931         if (elanDpns != null) {
 932             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
 933             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 934                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
 935                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
 936                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 937                     List<Action> listAction = new ArrayList<>();
 938                     int actionKey = 0;
 939                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
 940                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
 941                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 942                     bucketId++;
 943                     for (DpnInterfaces otherFes : dpnInterfaces) {
 944                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
 945                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
 946                             && !otherFes.getInterfaces().isEmpty()) {
 947                             try {
 948                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
 949                                         dpnInterface.getDpId(), otherFes.getDpId(),
 950                                         elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId()
 951                                                 : elanTag);
 952                                 if (!remoteListActionInfo.isEmpty()) {
 953                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
 954                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 955                                     bucketId++;
 956                                 }
 957                             } catch (Exception ex) {
 958                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
 959                                         + "Logical Group Interface not found between source Dpn - {}, "
 960                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
 961                                 return;
 962                             }
 963                         }
 964                     }
 965                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
 966                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
 967                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
 968
 969                     if (remoteListBucketInfo.isEmpty()) {
 970                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
 971                         continue;
 972                     }
 973                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 974                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
 975                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
 976                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
 977                 }
 978             }
 979             try {
 980                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 981             } catch (InterruptedException e1) {
 982                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
 983             }
 984         }
 985     }
 986
 987     /**
 988      * Returns the bucket info with the given interface as the only bucket.
 989      */
 990     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
 991         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
 992                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
 993     }
 994
 995     private List<MatchInfo> buildMatchesForVni(Long vni) {
 996         List<MatchInfo> mkMatches = new ArrayList<>();
 997         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
 998         mkMatches.add(match);
 999         return mkMatches;
1000     }
1001
1002     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1003         List<InstructionInfo> mkInstructions = new ArrayList<>();
1004         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1005         return mkInstructions;
1006     }
1007
1008     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1009         List<MatchInfo> mkMatches = new ArrayList<>();
1010         // Matching metadata
1011         mkMatches.add(new MatchMetadata(
1012                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1013         return mkMatches;
1014     }
1015
1016     /**
1017      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1018      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1019      *
1020      * @param elanTag
1021      *            elanTag to be written in metadata when flow is selected
1022      * @return the instructions ready to be installed in a flow
1023      */
1024     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1025         List<InstructionInfo> mkInstructions = new ArrayList<>();
1026         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1027                 .getElanMetadataMask()));
1028         /* applicable for EXTERNAL_TUNNEL_TABLE only
1029         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1030         */
1031         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1032         return mkInstructions;
1033     }
1034
1035     // Install DMAC entry on dst DPN
1036     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1037             BigInteger dstDpId) throws ElanException {
1038         String interfaceName = interfaceInfo.getInterfaceName();
1039         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1040         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1041             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1042             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1043             for (MacEntry macEntry : macEntries) {
1044                 String macAddress = macEntry.getMacAddress().getValue();
1045                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1046                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1047                         interfaceInfo.getDpId())) {
1048                     LOG.info("Acquired lock for mac : " + macAddress + ". Proceeding with remote dmac"
1049                             + " install operation.");
1050                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1051                             writeFlowTx);
1052                 }
1053             }
1054             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1055         }
1056         return Collections.emptyList();
1057     }
1058
1059     private void createDropBucket(List<Bucket> listBucket) {
1060         List<Action> actionsInfos = new ArrayList<>();
1061         actionsInfos.add(new ActionDrop().buildAction());
1062         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1063                 MDSALUtil.WATCH_GROUP);
1064         listBucket.add(dropBucket);
1065     }
1066
1067     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1068             InterfaceInfo interfaceInfo) {
1069         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1070         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1071     }
1072
1073     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1074             InterfaceInfo interfaceInfo) {
1075         List<Bucket> listBucket = new ArrayList<>();
1076         int bucketId = 0;
1077         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1078
1079         List<String> interfaces = new ArrayList<>();
1080         if (newDpnInterface != null) {
1081             interfaces = newDpnInterface.getInterfaces();
1082         }
1083         for (String ifName : interfaces) {
1084             // In case if there is a InterfacePort in the cache which is not in
1085             // operational state, skip processing it
1086             InterfaceInfo ifInfo = interfaceManager
1087                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1088             if (!isOperational(ifInfo)) {
1089                 continue;
1090             }
1091
1092             if (!interfaceManager.isExternalInterface(ifName)) {
1093                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1094                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1095                 bucketId++;
1096             }
1097         }
1098
1099         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1100                 MDSALUtil.buildBucketLists(listBucket));
1101         LOG.trace("installing the localBroadCast Group:{}", group);
1102         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1103     }
1104
1105     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1106             InterfaceInfo interfaceInfo) {
1107         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1108         if (etreeInstance != null) {
1109             List<Bucket> listBucket = new ArrayList<>();
1110             int bucketId = 0;
1111
1112             List<String> interfaces = new ArrayList<>();
1113             if (newDpnInterface != null) {
1114                 interfaces = newDpnInterface.getInterfaces();
1115             }
1116             for (String ifName : interfaces) {
1117                 // In case if there is a InterfacePort in the cache which is not
1118                 // in
1119                 // operational state, skip processing it
1120                 InterfaceInfo ifInfo = interfaceManager
1121                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1122                 if (!isOperational(ifInfo)) {
1123                     continue;
1124                 }
1125
1126                 if (!interfaceManager.isExternalInterface(ifName)) {
1127                     // only add root interfaces
1128                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1129                 }
1130             }
1131
1132             if (listBucket.isEmpty()) { // No Buckets
1133                 createDropBucket(listBucket);
1134             }
1135
1136             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1137             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1138             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1139                     MDSALUtil.buildBucketLists(listBucket));
1140             LOG.trace("installing the localBroadCast Group:{}", group);
1141             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1142         }
1143     }
1144
1145     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1146             InterfaceInfo ifInfo) {
1147         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1148         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1149             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1150                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1151             bucketId++;
1152         }
1153         return bucketId;
1154     }
1155
1156     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1157             WriteTransaction deleteFlowGroupTx) {
1158         BigInteger dpnId = interfaceInfo.getDpId();
1159         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1160         List<Bucket> listBuckets = new ArrayList<>();
1161         int bucketId = 0;
1162         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1163         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1164         // interfaceInfo));
1165         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1166                 MDSALUtil.buildBucketLists(listBuckets));
1167         LOG.trace("deleted the localBroadCast Group:{}", group);
1168         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1169     }
1170
1171     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1172             WriteTransaction deleteFlowGroupTx) {
1173         int bucketId = 0;
1174         int actionKey = 0;
1175         Long elanTag = elanInfo.getElanTag();
1176         List<Bucket> listBuckets = new ArrayList<>();
1177         List<Action> listAction = new ArrayList<>();
1178         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1179         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1180                 MDSALUtil.WATCH_GROUP));
1181         bucketId++;
1182         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1183         BigInteger dpnId = interfaceInfo.getDpId();
1184         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1185         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1186                 MDSALUtil.buildBucketLists(listBuckets));
1187         LOG.trace("deleting the remoteBroadCast group:{}", group);
1188         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1189     }
1190
1191     /**
1192      * Installs a flow in the External Tunnel table consisting in translating
1193      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1194      * elanTag that will be used later in the ELANs pipeline.
1195      *
1196      * @param dpnId
1197      *            the dpn id
1198      * @param elanInfo
1199      *            the elan info
1200      */
1201     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1202         long elanTag = elanInfo.getElanTag();
1203         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1204                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1205                 elanInfo.getElanInstanceName(), // flowName
1206                 0, // idleTimeout
1207                 0, // hardTimeout
1208                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1209                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1210                 getInstructionsIntOrExtTunnelTable(elanTag));
1211
1212         mdsalManager.installFlow(flowEntity);
1213     }
1214
1215     /**
1216      * Removes, from External Tunnel table, the flow that translates from VNI to
1217      * elanTag. Important: ensure this method is only called whenever there is
1218      * no other ElanInterface in the specified DPN
1219      *
1220      * @param dpnId
1221      *            DPN whose Ext Tunnel table is going to be modified
1222      * @param elanInfo
1223      *            holds the elanTag needed for selecting the flow to be removed
1224      */
1225     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1226         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1227         // last ElanInstance plus
1228         // adding a new one does (almost at the same time) are executed in that
1229         // exact order
1230
1231         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1232         FlowEntity flowEntity = new FlowEntityBuilder()
1233             .setDpnId(dpnId)
1234             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1235             .setFlowId(flowId)
1236             .build();
1237         mdsalManager.removeFlow(flowEntity);
1238     }
1239
1240     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1241         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1242         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1243     }
1244
1245     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1246             WriteTransaction writeFlowGroupTx) {
1247         List<? extends MatchInfoBase> listMatchInfoBase;
1248         List<InstructionInfo> instructionInfos;
1249         long serviceId;
1250         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1251             serviceId = elanTag;
1252             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1253             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1254         } else {
1255             serviceId = elanInfo.getSegmentationId();
1256             listMatchInfoBase = buildMatchesForVni(serviceId);
1257             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1258         }
1259         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1260                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1261                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1262                 instructionInfos);
1263         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1264     }
1265
1266     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1267             WriteTransaction writeFlowGroupTx) {
1268         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1269         if (etreeInstance != null) {
1270             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1271         }
1272     }
1273
1274     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1275         long elanTag = elanInfo.getElanTag();
1276         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1277         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1278         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1279     }
1280
1281     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1282             WriteTransaction writeFlowGroupTx) {
1283         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1284         if (etreeLeafTag != null) {
1285             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1286             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1287             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1288         }
1289     }
1290
1291     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1292             WriteTransaction writeFlowGroupTx) {
1293         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1294                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1295                 5, elanInfo.getElanInstanceName(), 0, 0,
1296                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1297                 getMatchesForElanTag(elanTag, /* SH flag */false),
1298                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1299
1300         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1301     }
1302
1303     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1304             WriteTransaction writeFlowGroupTx) {
1305         // only if ELAN can connect to external network, perform the following
1306
1307         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1308             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1309                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1310                     5, elanInfo.getElanInstanceName(), 0, 0,
1311                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1312                     getMatchesForElanTag(elanTag, /* SH flag */true),
1313                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1314             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1315         }
1316     }
1317
1318
1319     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1320             long elanTag) {
1321         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1322                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1323         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1324
1325         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1326             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1327                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1328                     .build();
1329             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1330         }
1331     }
1332
1333     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1334         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1335     }
1336
1337     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1338         if (isStandardElanService(elanInterface)) {
1339             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1340                     elanInterface.getName(), lportTag, tx);
1341         } else { // Etree service
1342             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1343         }
1344     }
1345
1346     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1347             WriteTransaction tx) {
1348         int instructionKey = 0;
1349         List<Instruction> instructions = new ArrayList<>();
1350         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1351                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1352
1353         List<Action> actions = new ArrayList<>();
1354         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1355                 lportTag).buildAction());
1356         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1357                 elanTag).buildAction());
1358         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1359
1360         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1361                 ++instructionKey));
1362
1363         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1364         BoundServices serviceInfo = ElanUtils.getBoundServices(
1365                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1366                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1367         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1368         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1369                 bindServiceId);
1370         if (!existingElanService.isPresent()) {
1371             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1372                     WriteTransaction.CREATE_MISSING_PARENTS);
1373         }
1374     }
1375
1376     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1377             WriteTransaction tx) {
1378         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1379             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1380                     lportTag, tx);
1381         } else {
1382             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1383             if (etreeInstance == null) {
1384                 LOG.error("EtreeInterface " + elanInterface.getName() + " is associated with a non EtreeInstance: "
1385                         + elanInfo.getElanInstanceName());
1386             } else {
1387                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1388                         elanInterface.getName(), lportTag, tx);
1389             }
1390         }
1391     }
1392
1393     private boolean isStandardElanService(ElanInterface elanInterface) {
1394         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1395     }
1396
1397     protected void unbindService(String interfaceName, ReadWriteTransaction tx) throws ReadFailedException {
1398         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1399         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1400         if (tx.read(LogicalDatastoreType.CONFIGURATION, bindServiceId).checkedGet().isPresent()) {
1401             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1402         }
1403     }
1404
1405     private String getFlowRef(long tableId, long elanTag) {
1406         return String.valueOf(tableId) + elanTag;
1407     }
1408
1409     private String getFlowRef(long tableId, long elanTag, String flowName) {
1410         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1411                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1412     }
1413
1414     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1415         return String.valueOf(tableId) + elanTag + shFlag;
1416     }
1417
1418     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1419         List<Action> listAction = new ArrayList<>();
1420         int actionKey = 0;
1421         listAction.add(
1422             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1423         actionKey++;
1424         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1425         return listAction;
1426     }
1427
1428     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1429             List<String> interfaceNames, WriteTransaction tx) {
1430         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1431                 .setKey(new DpnInterfacesKey(dpId)).build();
1432         tx.put(LogicalDatastoreType.OPERATIONAL,
1433                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1434                 WriteTransaction.CREATE_MISSING_PARENTS);
1435         return dpnInterface;
1436     }
1437
1438     /**
1439      * Delete elan dpn interface from operational DS.
1440      *
1441      * @param elanInstanceName
1442      *            the elan instance name
1443      * @param dpId
1444      *            the dp id
1445      */
1446     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1447         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1448                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1449         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1450                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1451         if (dpnInterfaces.isPresent()) {
1452             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1453         }
1454     }
1455
1456     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1457             WriteTransaction tx) {
1458         List<String> interfaceNames = new ArrayList<>();
1459         interfaceNames.add(interfaceName);
1460         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1461                 .setKey(new DpnInterfacesKey(dpId)).build();
1462         tx.put(LogicalDatastoreType.OPERATIONAL,
1463                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1464                 WriteTransaction.CREATE_MISSING_PARENTS);
1465         return dpnInterface;
1466     }
1467
1468     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1469         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1470                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1471         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1472                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1473         // Adding new Elan Interface Port to the operational DataStore without
1474         // Static-Mac Entries..
1475         if (!interfaceMacTables.isPresent()) {
1476             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1477                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1478             tx.put(LogicalDatastoreType.OPERATIONAL,
1479                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1480                     WriteTransaction.CREATE_MISSING_PARENTS);
1481         }
1482     }
1483
1484     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1485         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1486         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1487                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1488         // Adding new Elan Interface Port to the operational DataStore without
1489         // Static-Mac Entries..
1490         if (elanInterfaceLists.isPresent()) {
1491             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1492             if (interfaceLists == null) {
1493                 interfaceLists = new ArrayList<>();
1494             }
1495             interfaceLists.add(interfaceName);
1496             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1497                     .setKey(new ElanKey(elanInstanceName)).build();
1498             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1499                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1500         }
1501     }
1502
1503     private boolean isOperational(InterfaceInfo interfaceInfo) {
1504         if (interfaceInfo == null) {
1505             return false;
1506         }
1507         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1508     }
1509
1510     @SuppressWarnings("checkstyle:IllegalCatch")
1511     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1512         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1513         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1514                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1515         if (dpnInterfaceLists == null) {
1516             return;
1517         }
1518         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1519         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1520             int cnt = 0;
1521             String elanName = elanDpns.getElanInstanceName();
1522             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1523             if (elanInfo == null) {
1524                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1525                         + "skipping this ELAN for tunnel handling", elanName);
1526                 continue;
1527             }
1528             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1529                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1530                 continue;
1531             }
1532             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1533             if (dpnInterfaces == null) {
1534                 continue;
1535             }
1536             DpnInterfaces dstDpnIf = null;
1537             for (DpnInterfaces dpnIf : dpnInterfaces) {
1538                 BigInteger dpnIfDpId = dpnIf.getDpId();
1539                 if (dpnIfDpId.equals(srcDpId)) {
1540                     cnt++;
1541                 } else if (dpnIfDpId.equals(dstDpId)) {
1542                     cnt++;
1543                     dstDpnIf = dpnIf;
1544                 }
1545             }
1546             if (cnt == 2) {
1547                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1548                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1549                 jobCoordinator.enqueueJob(elanName, () -> {
1550                     // update Remote BC Group
1551                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1552                     try {
1553                         elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId);
1554                     } catch (RuntimeException e) {
1555                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1556                     }
1557                     Set<String> interfaceLists = new HashSet<>();
1558                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1559                     for (String ifName : interfaceLists) {
1560                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1561                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1562                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1563                             if (isOperational(interfaceInfo)) {
1564                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1565                             }
1566                             return Collections.emptyList();
1567                         }, ElanConstants.JOB_MAX_RETRIES);
1568                     }
1569                     return Collections.emptyList();
1570                 }, ElanConstants.JOB_MAX_RETRIES);
1571             }
1572
1573         }
1574     }
1575
1576     /**
1577      * Handle external tunnel state event.
1578      *
1579      * @param externalTunnel
1580      *            the external tunnel
1581      * @param intrf
1582      *            the interface
1583      * @throws ElanException in case of issues creating the flow objects
1584      */
1585     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1586         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1587             return;
1588         }
1589         // dpId/externalNodeId will be available either in source or destination
1590         // based on the tunnel end point
1591         BigInteger dpId = null;
1592         NodeId externalNodeId = null;
1593         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1594             dpId = new BigInteger(externalTunnel.getSourceDevice());
1595             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1596         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1597             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1598             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1599         }
1600         if (dpId == null || externalNodeId == null) {
1601             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1602             return;
1603         }
1604
1605         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1606         if (dpnInterfaceLists == null) {
1607             return;
1608         }
1609         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1610         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1611             String elanName = elanDpns.getElanInstanceName();
1612             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1613
1614             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1615             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1616                     || dpnInterfaces.getInterfaces().isEmpty()) {
1617                 continue;
1618             }
1619             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1620
1621             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpId);
1622             // install L2gwDevices local macs in dpn.
1623             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1624             // Install dpn macs on external device
1625             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1626                     externalNodeId);
1627         }
1628         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1629     }
1630
1631     /**
1632      * Installs dpn macs in external device. first it checks if the physical
1633      * locator towards this dpn tep is present or not if the physical locator is
1634      * present go ahead and add the ucast macs otherwise update the mcast mac
1635      * entry to include this dpn tep ip and schedule the job to put ucast macs
1636      * once the physical locator is programmed in device
1637      *
1638      * @param elanName
1639      *            the elan name
1640      * @param lstElanInterfaceNames
1641      *            the lst Elan interface names
1642      * @param dpnId
1643      *            the dpn id
1644      * @param externalNodeId
1645      *            the external node id
1646      */
1647     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1648             NodeId externalNodeId) {
1649         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1650                 externalNodeId.getValue());
1651         if (elanL2GwDevice == null) {
1652             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1653             return;
1654         }
1655         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1656         if (dpnTepIp == null) {
1657             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1658                     externalNodeId);
1659             return;
1660         }
1661
1662         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1663         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1664                 LogicalDatastoreType.OPERATIONAL);
1665         boolean phyLocAlreadyExists =
1666                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1667                 dpnTepIp);
1668         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1669                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1670         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1671
1672         if (phyLocAlreadyExists) {
1673             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1674             return;
1675         }
1676         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1677         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1678     }
1679
1680     /**
1681      * Validate external tunnel state event.
1682      *
1683      * @param externalTunnel
1684      *            the external tunnel
1685      * @param intrf
1686      *            the intrf
1687      * @return true, if successful
1688      */
1689     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1690         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1691             String srcDevice = externalTunnel.getDestinationDevice();
1692             String destDevice = externalTunnel.getSourceDevice();
1693             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1694                     LogicalDatastoreType.CONFIGURATION);
1695             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1696                     otherEndPointExtTunnel);
1697             if (otherEndPointExtTunnel != null) {
1698                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1699                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1700                 if (otherEndPointInterfaceOperational) {
1701                     return true;
1702                 } else {
1703                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1704                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1705                 }
1706             }
1707         }
1708         return false;
1709     }
1710
1711     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1712         List<MatchInfo> mkMatches = new ArrayList<>();
1713         // Matching metadata
1714         mkMatches.add(
1715                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1716         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1717         return mkMatches;
1718     }
1719
1720     @Override
1721     protected ElanInterfaceManager getDataTreeChangeListener() {
1722         return this;
1723     }
1724
1725     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1726                                              BigInteger dpId) {
1727         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1728         elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1729         try {
1730             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1731         } catch (InterruptedException e) {
1732             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1733         }
1734     }
1735 }