elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlan;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashSet;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.Queue;
  25 import java.util.Set;
  26 import java.util.concurrent.ConcurrentHashMap;
  27 import java.util.concurrent.ConcurrentLinkedQueue;
  28 import java.util.stream.Collectors;
  29 import javax.annotation.PostConstruct;
  30 import javax.inject.Inject;
  31 import javax.inject.Singleton;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  34 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  35 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  36 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  37 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  38 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  39 import org.opendaylight.genius.itm.globals.ITMConstants;
  40 import org.opendaylight.genius.mdsalutil.FlowEntity;
  41 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  42 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  43 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  44 import org.opendaylight.genius.mdsalutil.MatchInfo;
  45 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  46 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  54 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  55 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  56 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  57 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  58 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  59 import org.opendaylight.genius.utils.ServiceIndex;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.netvirt.elan.ElanException;
  62 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  63 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  64 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  65 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  66 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  67 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  68 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  69 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  70 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  71 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  72 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  73 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  74 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  75 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 119 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 /**
 125  * Class in charge of handling creations, modifications and removals of
 126  * ElanInterfaces.
 127  *
 128  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 129  */
 130 @Singleton
 131 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager> {
 132     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 133     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 134     private static final boolean SH_FLAG_SET = true;
 135     private static final boolean SH_FLAG_UNSET = false;
 136
 137     private final DataBroker broker;
 138     private final IMdsalApiManager mdsalManager;
 139     private final IInterfaceManager interfaceManager;
 140     private final IdManagerService idManager;
 141     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 142     private final INeutronVpnManager neutronVpnManager;
 143     private final ElanItmUtils elanItmUtils;
 144     private final ElanEtreeUtils elanEtreeUtils;
 145     private final ElanL2GatewayUtils elanL2GatewayUtils;
 146     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 147     private final ElanUtils elanUtils;
 148     private final JobCoordinator jobCoordinator;
 149     private final ElanInstanceCache elanInstanceCache;
 150     private final ElanInterfaceCache elanInterfaceCache;
 151
 152     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 153         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 154
 155     @Inject
 156     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 157                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 158                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 159                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 160                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 161                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 162                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 163                                 final ElanInstanceCache elanInstanceCache,
 164                                 final ElanInterfaceCache elanInterfaceCache) {
 165         super(ElanInterface.class, ElanInterfaceManager.class);
 166         this.broker = dataBroker;
 167         this.idManager = managerService;
 168         this.mdsalManager = mdsalApiManager;
 169         this.interfaceManager = interfaceManager;
 170         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 171         this.neutronVpnManager = neutronVpnManager;
 172         this.elanItmUtils = elanItmUtils;
 173         this.elanEtreeUtils = elanEtreeUtils;
 174         this.elanL2GatewayUtils = elanL2GatewayUtils;
 175         this.elanUtils = elanUtils;
 176         this.jobCoordinator = jobCoordinator;
 177         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 178         this.elanInstanceCache = elanInstanceCache;
 179         this.elanInterfaceCache = elanInterfaceCache;
 180     }
 181
 182     @Override
 183     @PostConstruct
 184     public void init() {
 185         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 186     }
 187
 188     @Override
 189     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 190         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 191     }
 192
 193     @Override
 194     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 195         String interfaceName = del.getName();
 196         ElanInstance elanInfo = elanInstanceCache.get(del.getElanInstanceName()).orNull();
 197         /*
 198          * Handling in case the elan instance is deleted.If the Elan instance is
 199          * deleted, there is no need to explicitly delete the elan interfaces
 200          */
 201         if (elanInfo == null) {
 202             return;
 203         }
 204         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 205         if (interfaceInfo == null && elanInfo.isExternal()) {
 206             // In deleting external network, the underlying ietf Inteface might have been removed
 207             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 208             // from Operational DS instead
 209             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 210         }
 211         String elanInstanceName = elanInfo.getElanInstanceName();
 212         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 213                 interfaceName, interfaceInfo, this);
 214         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 215     }
 216
 217     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 218             InterfaceInfo interfaceInfo) {
 219         String elanName = elanInfo.getElanInstanceName();
 220         boolean isLastElanInterface = false;
 221         boolean isLastInterfaceOnDpn = false;
 222         BigInteger dpId = null;
 223         long elanTag = elanInfo.getElanTag();
 224         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 225         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 226         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 227         if (elanState == null) {
 228             interfaceTx.cancel();
 229             return Collections.emptyList();
 230         }
 231         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 232         List<String> elanInterfaces = elanState.getElanInterfaces();
 233         if (elanInterfaces.isEmpty()) {
 234             isLastElanInterface = true;
 235         }
 236         if (interfaceInfo != null) {
 237             dpId = interfaceInfo.getDpId();
 238             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 239                     interfaceName, elanTag, interfaceTx);
 240             /*
 241              * If there are not elan ports, remove the unknown dmac, terminating
 242              * service table flows, remote/local bc group
 243              */
 244             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 245                     || dpnInterfaces.getInterfaces().isEmpty()) {
 246                 // No more Elan Interfaces in this DPN
 247                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 248                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 249                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 250                 }
 251                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 252                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 253                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 254                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 255                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 256                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 257                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 258                         elanUtils.removeTerminatingServiceAction(dpId, elanInfo.getSegmentationId().intValue());
 259                     }
 260                     unsetExternalTunnelTable(dpId, elanInfo);
 261                 }
 262                 isLastInterfaceOnDpn = true;
 263             } else {
 264                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 265             }
 266         }
 267
 268         List<ListenableFuture<Void>> futures = new ArrayList<>();
 269         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 270         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 271
 272         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 273             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 274         }
 275         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 276                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 277         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 278                 ElanConstants.JOB_MAX_RETRIES);
 279
 280         return futures;
 281     }
 282
 283     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 284             WriteTransaction deleteFlowGroupTx) {
 285         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 286         if (etreeLeafTag != null) {
 287             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 288             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 289         }
 290     }
 291
 292     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 293             WriteTransaction deleteFlowGroupTx) {
 294         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 295         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 296     }
 297
 298     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 299             WriteTransaction deleteFlowGroupTx) {
 300         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 301         if (etreeInstance != null) {
 302             BigInteger dpnId = interfaceInfo.getDpId();
 303             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 304             List<Bucket> listBuckets = new ArrayList<>();
 305             int bucketId = 0;
 306             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 307             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 308                     MDSALUtil.buildBucketLists(listBuckets));
 309             LOG.trace("deleted the localBroadCast Group:{}", group);
 310             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 311         }
 312     }
 313
 314     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 315             WriteTransaction deleteFlowGroupTx) {
 316         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 317         if (etreeInstance != null) {
 318             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 319             int bucketId = 0;
 320             int actionKey = 0;
 321             List<Bucket> listBuckets = new ArrayList<>();
 322             List<Action> listAction = new ArrayList<>();
 323             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 324             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 325                     MDSALUtil.WATCH_GROUP));
 326             bucketId++;
 327             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 328             BigInteger dpnId = interfaceInfo.getDpId();
 329             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 330             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 331                     MDSALUtil.buildBucketLists(listBuckets));
 332             LOG.trace("deleting the remoteBroadCast group:{}", group);
 333             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 334         }
 335     }
 336
 337     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 338         String elanName = elanInfo.getElanInstanceName();
 339         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 340         if (elanState == null) {
 341             return elanState;
 342         }
 343         List<String> elanInterfaces = elanState.getElanInterfaces();
 344         boolean isRemoved = elanInterfaces.remove(interfaceName);
 345         if (!isRemoved) {
 346             return elanState;
 347         }
 348
 349         if (elanInterfaces.isEmpty()) {
 350             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 351             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 352             tx.delete(LogicalDatastoreType.OPERATIONAL,
 353                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 354         } else {
 355             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 356                     .setKey(new ElanKey(elanName)).build();
 357             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 358                     updateElanState);
 359         }
 360         return elanState;
 361     }
 362
 363     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 364         // removing the ElanInterface from the config data_store if interface is
 365         // not present in Interface config DS
 366         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 367                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 368             tx.delete(LogicalDatastoreType.CONFIGURATION,
 369                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 370         }
 371     }
 372
 373     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 374             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 375         String elanName = elanInfo.getElanInstanceName();
 376         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 377         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 378         InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 379                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 380         Optional<ElanInterfaceMac> existingElanInterfaceMac = ElanUtils.read(broker,
 381                 LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 382         LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 383         if (interfaceInfo != null) {
 384             if (existingElanInterfaceMac.isPresent()) {
 385                 List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 386                 List<MacEntry> macEntries = new ArrayList<>();
 387                 if (existingMacEntries != null && !existingMacEntries.isEmpty()) {
 388                     macEntries.addAll(existingMacEntries);
 389                 }
 390                 List<PhysAddress> macAddresses = macEntries.stream().map(macEntry -> {
 391                     PhysAddress macAddress = macEntry.getMacAddress();
 392                     LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 393                             macAddress.getValue(), interfaceName);
 394                     Optional<MacEntry> macEntryOptional = elanUtils.getMacEntryForElanInstance(elanName,
 395                             macAddress);
 396                     if (!isLastElanInterface && macEntryOptional.isPresent()) {
 397                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 398                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 399                     }
 400                     elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 401                     return macAddress;
 402                 }).collect(Collectors.toList());
 403
 404                 // Removing all those MACs from External Devices belonging
 405                 // to this ELAN
 406                 if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 407                     elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 408                 }
 409             }
 410             removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 411             removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 412         } else if (existingElanInterfaceMac.isPresent()) {
 413             // Interface does not exist in ConfigDS, so lets remove everything
 414             // about that interface related to Elan
 415             List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 416             if (macEntries != null) {
 417                 for (MacEntry macEntry : macEntries) {
 418                     PhysAddress macAddress = macEntry.getMacAddress();
 419                     if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 420                         interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 421                                 ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 422                     }
 423                 }
 424             }
 425         }
 426         if (existingElanInterfaceMac.isPresent()) {
 427             interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 428         }
 429         unbindService(interfaceName, interfaceTx);
 430         deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 431         List<ListenableFuture<Void>> futures = new ArrayList<>();
 432         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 433         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 434         return futures;
 435     }
 436
 437     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 438                                                                          String interfaceName, long elanTag,
 439                                                                          WriteTransaction tx) {
 440         DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 441         if (dpnInterfaces != null) {
 442             List<String> interfaceLists = dpnInterfaces.getInterfaces();
 443             if (interfaceLists != null) {
 444                 interfaceLists.remove(interfaceName);
 445             }
 446
 447             if (interfaceLists == null || interfaceLists.isEmpty()) {
 448                 deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 449                 deleteElanDpnInterface(elanName, dpId, tx);
 450             } else {
 451                 dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 452             }
 453         }
 454         return dpnInterfaces;
 455     }
 456
 457     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 458         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 459         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 460             BigInteger currentDpId = dpnInterface.getDpId();
 461             if (!currentDpId.equals(dpId)) {
 462                 for (String elanInterface : dpnInterface.getInterfaces()) {
 463                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 464                     if (macs == null || macs.getMacEntry() == null) {
 465                         continue;
 466                     }
 467                     for (MacEntry mac : macs.getMacEntry()) {
 468                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 469                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 470                     }
 471                 }
 472             }
 473         }
 474     }
 475
 476     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 477         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 478         if (etreeLeafTag != null) {
 479             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 480         }
 481     }
 482
 483     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 484         mdsalManager
 485                 .removeFlow(dpId,
 486                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 487                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 488                                         mac.getMacAddress().getValue(), elanTag)));
 489     }
 490
 491     /*
 492     * Possible Scenarios for update
 493     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 494         then all {1,2,3,4} should be removed
 495
 496         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 497         then all {1,2,3,4} should be added
 498
 499         c. if orig = {1,2,3,4} updated={2,3,4}
 500         then 1 should be removed
 501
 502         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 503         then we should just add 5
 504
 505         e. if orig = {1,2,3,4} updated={2,3,4,5}
 506         then 1 should be removed , 5 should be added
 507     * */
 508     @Override
 509     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 510         // updating the static-Mac Entries for the existing elanInterface
 511         String elanName = update.getElanInstanceName();
 512         String interfaceName = update.getName();
 513
 514         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 515         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 516         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 517         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 518
 519         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 520                 deletedEntry.getMacAddress()));
 521
 522         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 523         * Hence add the macentries for the same.*/
 524         for (StaticMacEntries staticMacEntry : updatedEntries) {
 525             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 526                     staticMacEntry.getMacAddress());
 527             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 528                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 529             WriteTransaction tx = broker.newWriteOnlyTransaction();
 530             if (existingMacEntry.isPresent()) {
 531                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 532                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 533                         tx);
 534             } else {
 535                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 536                         elanName, interfaceName, staticMacEntry, tx);
 537             }
 538             ElanUtils.waitForTransactionToComplete(tx);
 539         }
 540     }
 541
 542     @Override
 543     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 544         String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 545         String interfaceName = elanInterfaceAdded.getName();
 546         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 547         if (interfaceInfo == null) {
 548             LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 549             return;
 550         }
 551         ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 552
 553         if (elanInstance == null) {
 554             elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 555                     .setDescription(elanInterfaceAdded.getDescription()).build();
 556             // Add the ElanInstance in the Configuration data-store
 557             WriteTransaction tx = broker.newWriteOnlyTransaction();
 558             List<String> elanInterfaces = new ArrayList<>();
 559             elanInterfaces.add(interfaceName);
 560             elanInstance = ElanUtils.updateOperationalDataStore(broker, idManager,
 561                     elanInstance, elanInterfaces, tx);
 562             tx.submit();
 563         }
 564
 565         Long elanTag = elanInstance.getElanTag();
 566         // If elan tag is not updated, then put the elan interface into
 567         // unprocessed entry map and entry. Let entries
 568         // in this map get processed during ELAN update DCN.
 569         if (elanTag == null) {
 570             ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 571             if (elanInterfaces == null) {
 572                 elanInterfaces = new ConcurrentLinkedQueue<>();
 573             }
 574             elanInterfaces.add(elanInterfaceAdded);
 575             unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 576             return;
 577         }
 578         InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 579                 interfaceInfo, elanInstance, this);
 580         jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 581     }
 582
 583     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 584         List<ListenableFuture<Void>> futures = new ArrayList<>();
 585         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 586         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 587             return futures;
 588         }
 589         for (ElanInterface elanInterface : elanInterfaces) {
 590             String interfaceName = elanInterface.getName();
 591             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 592             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 593         }
 594         return futures;
 595     }
 596
 597     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 598             WriteTransaction writeFlowGroupTx) throws ElanException {
 599         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 600                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 601         List<DpnInterfaces> dpnInterfaceLists = null;
 602         if (elanDpnInterfacesList != null) {
 603             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 604         }
 605         if (dpnInterfaceLists == null) {
 606             dpnInterfaceLists = new ArrayList<>();
 607         }
 608         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 609             BigInteger dstDpId = interfaceInfo.getDpId();
 610             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 611                 continue;
 612             }
 613             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 614             for (String remoteIf : remoteElanInterfaces) {
 615                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 616                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 617                 if (elanIfMac == null) {
 618                     continue;
 619                 }
 620                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 621                 if (remoteMacEntries != null) {
 622                     for (MacEntry macEntry : remoteMacEntries) {
 623                         String macAddress = macEntry.getMacAddress().getValue();
 624                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 625                                 dstDpId, elanInstance.getElanInstanceName());
 626                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 627                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 628                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 629                     }
 630                 }
 631             }
 632         }
 633     }
 634
 635     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 636             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 637         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 638         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 639         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 640
 641         String interfaceName = elanInterface.getName();
 642         String elanInstanceName = elanInterface.getElanInstanceName();
 643
 644         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 645         WriteTransaction tx = broker.newWriteOnlyTransaction();
 646         if (elanInfo == null) {
 647             List<String> elanInterfaces = new ArrayList<>();
 648             elanInterfaces.add(interfaceName);
 649             ElanUtils.updateOperationalDataStore(broker, idManager,
 650                     elanInstance, elanInterfaces, tx);
 651         } else {
 652             createElanStateList(elanInstanceName, interfaceName, tx);
 653         }
 654         boolean isFirstInterfaceInDpn = false;
 655         // Specific actions to the DPN where the ElanInterface has been added,
 656         // for example, programming the
 657         // External tunnel table if needed or adding the ElanInterface to the
 658         // DpnInterfaces in the operational DS.
 659         BigInteger dpId = interfaceInfo.getDpId();
 660         DpnInterfaces dpnInterfaces = null;
 661         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 662             InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 663                     .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 664             Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 665                     LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 666             if (!existingElanDpnInterfaces.isPresent()) {
 667                 isFirstInterfaceInDpn = true;
 668                 // ELAN's 1st ElanInterface added to this DPN
 669                 dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 670                 // The 1st ElanInterface in a DPN must program the Ext Tunnel
 671                 // table, but only if Elan has VNI
 672                 if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 673                     setExternalTunnelTable(dpId, elanInstance);
 674                 }
 675                 elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 676             } else {
 677                 List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 678                 elanInterfaces.add(interfaceName);
 679                 if (elanInterfaces.size() == 1) { // 1st dpn interface
 680                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 681                 }
 682                 dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 683             }
 684         }
 685
 686         // add code to install Local/Remote BC group, unknow DMAC entry,
 687         // terminating service table flow entry
 688         // call bindservice of interfacemanager to create ingress table flow
 689         // enty.
 690         // Add interface to the ElanInterfaceForwardingEntires Container
 691         createElanInterfaceTablesList(interfaceName, tx);
 692         List<ListenableFuture<Void>> futures = new ArrayList<>();
 693         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 694         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces,
 695                 isFirstInterfaceInDpn, null);
 696
 697         // add the vlan provider interface to remote BC group for the elan
 698         // for internal vlan networks
 699         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 700             if (interfaceManager.isExternalInterface(interfaceName)) {
 701                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 702                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 703             }
 704         }
 705
 706         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 707             //update the remote-DPNs remoteBC group entry with Tunnels
 708             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 709             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 710         }
 711
 712         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 713         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 714                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 715         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 716         return futures;
 717     }
 718
 719     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 720             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 721             throws ElanException {
 722         String elanInstanceName = elanInstance.getElanInstanceName();
 723         String interfaceName = elanInterface.getName();
 724         WriteTransaction tx = broker.newWriteOnlyTransaction();
 725         BigInteger dpId = interfaceInfo.getDpId();
 726         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 727         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 728                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 729
 730         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 731         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 732
 733         boolean isInterfaceOperational = isOperational(interfaceInfo);
 734         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 735             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 736                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 737                         staticMacEntry.getMacAddress());
 738                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 739                         LogicalDatastoreType.OPERATIONAL, macId);
 740                 if (existingMacEntry.isPresent()) {
 741                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 742                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 743                             existingMacEntry.get(), tx);
 744                 } else {
 745                     elanForwardingEntriesHandler
 746                             .addElanInterfaceForwardingTableList(elanInstanceName, interfaceName, staticMacEntry, tx);
 747                 }
 748
 749                 if (isInterfaceOperational) {
 750                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 751                     // DPNs
 752                     String macAddress = staticMacEntry.getMacAddress().getValue();
 753                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 754                             macAddress, elanInstanceName, interfaceName);
 755                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 756                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 757                 }
 758             }
 759
 760             if (isInterfaceOperational) {
 761                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 762                 // on purpose.
 763                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 764                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 765                 }
 766                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 767                         staticMacAddresses);
 768             }
 769         }
 770         List<ListenableFuture<Void>> futures = new ArrayList<>();
 771         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 772         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 773         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 774             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 775             try {
 776                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 777                 if (neutronPort != null) {
 778                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 779                 }
 780             } catch (IllegalArgumentException ex) {
 781                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 782             }
 783         }
 784         return futures;
 785     }
 786
 787     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 788             PhysAddress physAddress) {
 789         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 790         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 791         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 792                 LogicalDatastoreType.OPERATIONAL, macId);
 793
 794         if (!existingMacEntry.isPresent()) {
 795             return;
 796         }
 797
 798         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 799                 .setKey(new MacEntryKey(physAddress)).build();
 800         WriteTransaction tx = broker.newWriteOnlyTransaction();
 801         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 802                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry, tx);
 803         ElanUtils.waitForTransactionToComplete(tx);
 804     }
 805
 806     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 807         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 808                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 809     }
 810
 811     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 812             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 813             WriteTransaction writeFlowGroupTx) throws ElanException {
 814         if (!isOperational(interfaceInfo)) {
 815             return;
 816         }
 817         BigInteger dpId = interfaceInfo.getDpId();
 818         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 819             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 820         }
 821         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 822         if (isFirstInterfaceInDpn) {
 823             // Terminating Service , UnknownDMAC Table.
 824             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 825             // for ELAN Instance is VxLAN
 826             if (isVxlan(elanInstance)) {
 827                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 828             }
 829             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 830             /*
 831              * Install remote DMAC flow. This is required since this DPN is
 832              * added later to the elan instance and remote DMACs of other
 833              * interfaces in this elan instance are not present in the current
 834              * dpn.
 835              */
 836             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 837                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 838                         elanInstance.getElanInstanceName());
 839                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 840             }
 841         }
 842         // bind the Elan service to the Interface
 843         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 844     }
 845
 846     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 847             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, WriteTransaction tx) {
 848         if (!isOperational(interfaceInfo)) {
 849             return;
 850         }
 851         // LocalBroadcast Group creation with elan-Interfaces
 852         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 853         if (isFirstInterfaceInDpn) {
 854             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 855             BigInteger dpId = interfaceInfo.getDpId();
 856             // RemoteBroadcast Group creation
 857             try {
 858                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 859             } catch (InterruptedException e1) {
 860                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 861             }
 862             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 863             try {
 864                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 865             } catch (InterruptedException e1) {
 866                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 867             }
 868         }
 869     }
 870
 871     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 872             WriteTransaction writeFlowGroupTx) {
 873         int ifTag = interfaceInfo.getInterfaceTag();
 874         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 875                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 876                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 877                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 878                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 879
 880         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 881
 882         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 883                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 884                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 885                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 886
 887         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 888     }
 889
 890     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 891             WriteTransaction deleteFlowGroupTx) {
 892         int ifTag = interfaceInfo.getInterfaceTag();
 893         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 894                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 895                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 896                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 897                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 898
 899         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flow, deleteFlowGroupTx);
 900
 901         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 902                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 903                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 904                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 905
 906         mdsalManager.removeFlowToTx(interfaceInfo.getDpId(), flowEntity, deleteFlowGroupTx);
 907     }
 908
 909     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 910                                                      InterfaceInfo interfaceInfo, long elanTag) {
 911         return elanL2GatewayMulticastUtils.getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(),
 912                 bucketKeyStart, elanTag);
 913     }
 914
 915     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 916         int elanTag = elanInfo.getElanTag().intValue();
 917         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 918         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 919         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
 920         if (etreeInstance != null) {
 921             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 922             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 923             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 924         }
 925     }
 926
 927     @SuppressWarnings("checkstyle:IllegalCatch")
 928     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
 929         int bucketId = 0;
 930         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
 931         if (elanDpns != null) {
 932             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
 933             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 934                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
 935                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
 936                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
 937                     List<Action> listAction = new ArrayList<>();
 938                     int actionKey = 0;
 939                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
 940                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
 941                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 942                     bucketId++;
 943                     for (DpnInterfaces otherFes : dpnInterfaces) {
 944                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
 945                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
 946                             && !otherFes.getInterfaces().isEmpty()) {
 947                             try {
 948                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
 949                                         dpnInterface.getDpId(), otherFes.getDpId(),
 950                                         elanUtils.isOpenstackVniSemanticsEnforced() ? elanInfo.getSegmentationId()
 951                                                 : elanTag);
 952                                 if (!remoteListActionInfo.isEmpty()) {
 953                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
 954                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
 955                                     bucketId++;
 956                                 }
 957                             } catch (Exception ex) {
 958                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
 959                                         + "Logical Group Interface not found between source Dpn - {}, "
 960                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
 961                                 return;
 962                             }
 963                         }
 964                     }
 965                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
 966                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
 967                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
 968
 969                     if (remoteListBucketInfo.isEmpty()) {
 970                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
 971                         continue;
 972                     }
 973                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 974                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
 975                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
 976                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
 977                 }
 978             }
 979             try {
 980                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 981             } catch (InterruptedException e1) {
 982                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
 983             }
 984         }
 985     }
 986
 987     /**
 988      * Returns the bucket info with the given interface as the only bucket.
 989      */
 990     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
 991         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
 992                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
 993     }
 994
 995     private List<MatchInfo> buildMatchesForVni(Long vni) {
 996         List<MatchInfo> mkMatches = new ArrayList<>();
 997         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
 998         mkMatches.add(match);
 999         return mkMatches;
1000     }
1001
1002     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1003         List<InstructionInfo> mkInstructions = new ArrayList<>();
1004         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1005         return mkInstructions;
1006     }
1007
1008     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1009         List<MatchInfo> mkMatches = new ArrayList<>();
1010         // Matching metadata
1011         mkMatches.add(new MatchMetadata(
1012                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1013         return mkMatches;
1014     }
1015
1016     /**
1017      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1018      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1019      *
1020      * @param elanTag
1021      *            elanTag to be written in metadata when flow is selected
1022      * @return the instructions ready to be installed in a flow
1023      */
1024     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1025         List<InstructionInfo> mkInstructions = new ArrayList<>();
1026         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1027                 .getElanMetadataMask()));
1028         /* applicable for EXTERNAL_TUNNEL_TABLE only
1029         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1030         */
1031         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1032         return mkInstructions;
1033     }
1034
1035     // Install DMAC entry on dst DPN
1036     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1037             BigInteger dstDpId) throws ElanException {
1038         String interfaceName = interfaceInfo.getInterfaceName();
1039         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1040         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1041             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1042             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1043             for (MacEntry macEntry : macEntries) {
1044                 String macAddress = macEntry.getMacAddress().getValue();
1045                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1046                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1047                         interfaceInfo.getDpId())) {
1048                     LOG.info("Acquired lock for mac : " + macAddress + ". Proceeding with remote dmac"
1049                             + " install operation.");
1050                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1051                             writeFlowTx);
1052                 }
1053             }
1054             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1055         }
1056         return Collections.emptyList();
1057     }
1058
1059     private void createDropBucket(List<Bucket> listBucket) {
1060         List<Action> actionsInfos = new ArrayList<>();
1061         actionsInfos.add(new ActionDrop().buildAction());
1062         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1063                 MDSALUtil.WATCH_GROUP);
1064         listBucket.add(dropBucket);
1065     }
1066
1067     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1068             InterfaceInfo interfaceInfo) {
1069         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1070         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1071     }
1072
1073     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1074             InterfaceInfo interfaceInfo) {
1075         List<Bucket> listBucket = new ArrayList<>();
1076         int bucketId = 0;
1077         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1078
1079         List<String> interfaces = new ArrayList<>();
1080         if (newDpnInterface != null) {
1081             interfaces = newDpnInterface.getInterfaces();
1082         }
1083         for (String ifName : interfaces) {
1084             // In case if there is a InterfacePort in the cache which is not in
1085             // operational state, skip processing it
1086             InterfaceInfo ifInfo = interfaceManager
1087                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1088             if (!isOperational(ifInfo)) {
1089                 continue;
1090             }
1091
1092             if (!interfaceManager.isExternalInterface(ifName)) {
1093                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1094                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1095                 bucketId++;
1096             }
1097         }
1098
1099         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1100                 MDSALUtil.buildBucketLists(listBucket));
1101         LOG.trace("installing the localBroadCast Group:{}", group);
1102         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1103     }
1104
1105     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1106             InterfaceInfo interfaceInfo) {
1107         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1108         if (etreeInstance != null) {
1109             List<Bucket> listBucket = new ArrayList<>();
1110             int bucketId = 0;
1111
1112             List<String> interfaces = new ArrayList<>();
1113             if (newDpnInterface != null) {
1114                 interfaces = newDpnInterface.getInterfaces();
1115             }
1116             for (String ifName : interfaces) {
1117                 // In case if there is a InterfacePort in the cache which is not
1118                 // in
1119                 // operational state, skip processing it
1120                 InterfaceInfo ifInfo = interfaceManager
1121                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1122                 if (!isOperational(ifInfo)) {
1123                     continue;
1124                 }
1125
1126                 if (!interfaceManager.isExternalInterface(ifName)) {
1127                     // only add root interfaces
1128                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1129                 }
1130             }
1131
1132             if (listBucket.isEmpty()) { // No Buckets
1133                 createDropBucket(listBucket);
1134             }
1135
1136             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1137             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1138             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1139                     MDSALUtil.buildBucketLists(listBucket));
1140             LOG.trace("installing the localBroadCast Group:{}", group);
1141             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1142         }
1143     }
1144
1145     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1146             InterfaceInfo ifInfo) {
1147         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1148         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1149             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1150                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1151             bucketId++;
1152         }
1153         return bucketId;
1154     }
1155
1156     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1157             WriteTransaction deleteFlowGroupTx) {
1158         BigInteger dpnId = interfaceInfo.getDpId();
1159         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1160         List<Bucket> listBuckets = new ArrayList<>();
1161         int bucketId = 0;
1162         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1163         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1164         // interfaceInfo));
1165         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1166                 MDSALUtil.buildBucketLists(listBuckets));
1167         LOG.trace("deleted the localBroadCast Group:{}", group);
1168         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1169     }
1170
1171     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1172             WriteTransaction deleteFlowGroupTx) {
1173         int bucketId = 0;
1174         int actionKey = 0;
1175         Long elanTag = elanInfo.getElanTag();
1176         List<Bucket> listBuckets = new ArrayList<>();
1177         List<Action> listAction = new ArrayList<>();
1178         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1179         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1180                 MDSALUtil.WATCH_GROUP));
1181         bucketId++;
1182         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1183         BigInteger dpnId = interfaceInfo.getDpId();
1184         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1185         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1186                 MDSALUtil.buildBucketLists(listBuckets));
1187         LOG.trace("deleting the remoteBroadCast group:{}", group);
1188         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1189     }
1190
1191     /**
1192      * Installs a flow in the External Tunnel table consisting in translating
1193      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1194      * elanTag that will be used later in the ELANs pipeline.
1195      *
1196      * @param dpnId
1197      *            the dpn id
1198      * @param elanInfo
1199      *            the elan info
1200      */
1201     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1202         long elanTag = elanInfo.getElanTag();
1203         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1204                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1205                 elanInfo.getElanInstanceName(), // flowName
1206                 0, // idleTimeout
1207                 0, // hardTimeout
1208                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1209                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1210                 getInstructionsIntOrExtTunnelTable(elanTag));
1211
1212         mdsalManager.installFlow(flowEntity);
1213     }
1214
1215     /**
1216      * Removes, from External Tunnel table, the flow that translates from VNI to
1217      * elanTag. Important: ensure this method is only called whenever there is
1218      * no other ElanInterface in the specified DPN
1219      *
1220      * @param dpnId
1221      *            DPN whose Ext Tunnel table is going to be modified
1222      * @param elanInfo
1223      *            holds the elanTag needed for selecting the flow to be removed
1224      */
1225     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1226         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1227         // last ElanInstance plus
1228         // adding a new one does (almost at the same time) are executed in that
1229         // exact order
1230
1231         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1232         FlowEntity flowEntity = new FlowEntityBuilder()
1233             .setDpnId(dpnId)
1234             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1235             .setFlowId(flowId)
1236             .build();
1237         mdsalManager.removeFlow(flowEntity);
1238     }
1239
1240     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1241         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1242         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1243     }
1244
1245     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1246             WriteTransaction writeFlowGroupTx) {
1247         List<? extends MatchInfoBase> listMatchInfoBase;
1248         List<InstructionInfo> instructionInfos;
1249         long serviceId;
1250         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1251             serviceId = elanTag;
1252             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1253             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1254         } else {
1255             serviceId = elanInfo.getSegmentationId();
1256             listMatchInfoBase = buildMatchesForVni(serviceId);
1257             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1258         }
1259         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1260                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1261                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1262                 instructionInfos);
1263         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1264     }
1265
1266     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1267             WriteTransaction writeFlowGroupTx) {
1268         EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1269         if (etreeInstance != null) {
1270             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1271         }
1272     }
1273
1274     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1275         long elanTag = elanInfo.getElanTag();
1276         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1277         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1278         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1279     }
1280
1281     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1282             WriteTransaction writeFlowGroupTx) {
1283         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1284         if (etreeLeafTag != null) {
1285             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1286             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1287             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1288         }
1289     }
1290
1291     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1292             WriteTransaction writeFlowGroupTx) {
1293         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1294                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1295                 5, elanInfo.getElanInstanceName(), 0, 0,
1296                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1297                 getMatchesForElanTag(elanTag, /* SH flag */false),
1298                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1299
1300         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1301     }
1302
1303     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1304             WriteTransaction writeFlowGroupTx) {
1305         // only if ELAN can connect to external network, perform the following
1306
1307         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1308             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1309                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1310                     5, elanInfo.getElanInstanceName(), 0, 0,
1311                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1312                     getMatchesForElanTag(elanTag, /* SH flag */true),
1313                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1314             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1315         }
1316     }
1317
1318
1319     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1320             long elanTag) {
1321         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1322                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1323         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1324
1325         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1326             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1327                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1328                     .build();
1329             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1330         }
1331     }
1332
1333     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1334         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1335     }
1336
1337     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1338         if (isStandardElanService(elanInterface)) {
1339             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1340                     elanInterface.getName(), lportTag, tx);
1341         } else { // Etree service
1342             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1343         }
1344     }
1345
1346     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1347             WriteTransaction tx) {
1348         int instructionKey = 0;
1349         List<Instruction> instructions = new ArrayList<>();
1350         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1351                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1352
1353         List<Action> actions = new ArrayList<>();
1354         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1355                 lportTag).buildAction());
1356         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1357                 elanTag).buildAction());
1358         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1359
1360         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1361                 ++instructionKey));
1362
1363         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1364         BoundServices serviceInfo = ElanUtils.getBoundServices(
1365                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1366                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1367         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1368         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1369                 bindServiceId);
1370         if (!existingElanService.isPresent()) {
1371             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1372                     WriteTransaction.CREATE_MISSING_PARENTS);
1373         }
1374     }
1375
1376     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1377             WriteTransaction tx) {
1378         if (elanInterface.getAugmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1379             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1380                     lportTag, tx);
1381         } else {
1382             EtreeInstance etreeInstance = elanInfo.getAugmentation(EtreeInstance.class);
1383             if (etreeInstance == null) {
1384                 LOG.error("EtreeInterface " + elanInterface.getName() + " is associated with a non EtreeInstance: "
1385                         + elanInfo.getElanInstanceName());
1386             } else {
1387                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1388                         elanInterface.getName(), lportTag, tx);
1389             }
1390         }
1391     }
1392
1393     private boolean isStandardElanService(ElanInterface elanInterface) {
1394         return elanInterface.getAugmentation(EtreeInterface.class) == null;
1395     }
1396
1397     protected void unbindService(String interfaceName, WriteTransaction tx) {
1398         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1399         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1400         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1401                 bindServiceId);
1402         if (existingElanService.isPresent()) {
1403             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1404         }
1405     }
1406
1407     private String getFlowRef(long tableId, long elanTag) {
1408         return String.valueOf(tableId) + elanTag;
1409     }
1410
1411     private String getFlowRef(long tableId, long elanTag, String flowName) {
1412         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1413                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1414     }
1415
1416     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1417         return String.valueOf(tableId) + elanTag + shFlag;
1418     }
1419
1420     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1421         List<Action> listAction = new ArrayList<>();
1422         int actionKey = 0;
1423         listAction.add(
1424             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1425         actionKey++;
1426         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1427         return listAction;
1428     }
1429
1430     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1431             List<String> interfaceNames, WriteTransaction tx) {
1432         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1433                 .setKey(new DpnInterfacesKey(dpId)).build();
1434         tx.put(LogicalDatastoreType.OPERATIONAL,
1435                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1436                 WriteTransaction.CREATE_MISSING_PARENTS);
1437         return dpnInterface;
1438     }
1439
1440     /**
1441      * Delete elan dpn interface from operational DS.
1442      *
1443      * @param elanInstanceName
1444      *            the elan instance name
1445      * @param dpId
1446      *            the dp id
1447      */
1448     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1449         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1450                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1451         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1452                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1453         if (dpnInterfaces.isPresent()) {
1454             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1455         }
1456     }
1457
1458     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1459             WriteTransaction tx) {
1460         List<String> interfaceNames = new ArrayList<>();
1461         interfaceNames.add(interfaceName);
1462         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1463                 .setKey(new DpnInterfacesKey(dpId)).build();
1464         tx.put(LogicalDatastoreType.OPERATIONAL,
1465                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1466                 WriteTransaction.CREATE_MISSING_PARENTS);
1467         return dpnInterface;
1468     }
1469
1470     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1471         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1472                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1473         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1474                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1475         // Adding new Elan Interface Port to the operational DataStore without
1476         // Static-Mac Entries..
1477         if (!interfaceMacTables.isPresent()) {
1478             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1479                     .setKey(new ElanInterfaceMacKey(interfaceName)).build();
1480             tx.put(LogicalDatastoreType.OPERATIONAL,
1481                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1482                     WriteTransaction.CREATE_MISSING_PARENTS);
1483         }
1484     }
1485
1486     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1487         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1488         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1489                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1490         // Adding new Elan Interface Port to the operational DataStore without
1491         // Static-Mac Entries..
1492         if (elanInterfaceLists.isPresent()) {
1493             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1494             if (interfaceLists == null) {
1495                 interfaceLists = new ArrayList<>();
1496             }
1497             interfaceLists.add(interfaceName);
1498             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1499                     .setKey(new ElanKey(elanInstanceName)).build();
1500             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1501                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1502         }
1503     }
1504
1505     private boolean isOperational(InterfaceInfo interfaceInfo) {
1506         if (interfaceInfo == null) {
1507             return false;
1508         }
1509         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1510     }
1511
1512     @SuppressWarnings("checkstyle:IllegalCatch")
1513     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) throws ElanException {
1514         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1515         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1516                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1517         if (dpnInterfaceLists == null) {
1518             return;
1519         }
1520         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1521         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1522             int cnt = 0;
1523             String elanName = elanDpns.getElanInstanceName();
1524             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1525             if (elanInfo == null) {
1526                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1527                         + "skipping this ELAN for tunnel handling", elanName);
1528                 continue;
1529             }
1530             if (ElanUtils.isFlat(elanInfo) || ElanUtils.isVlan(elanInfo)) {
1531                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1532                 continue;
1533             }
1534             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1535             if (dpnInterfaces == null) {
1536                 continue;
1537             }
1538             DpnInterfaces dstDpnIf = null;
1539             for (DpnInterfaces dpnIf : dpnInterfaces) {
1540                 BigInteger dpnIfDpId = dpnIf.getDpId();
1541                 if (dpnIfDpId.equals(srcDpId)) {
1542                     cnt++;
1543                 } else if (dpnIfDpId.equals(dstDpId)) {
1544                     cnt++;
1545                     dstDpnIf = dpnIf;
1546                 }
1547             }
1548             if (cnt == 2) {
1549                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1550                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1551                 jobCoordinator.enqueueJob(elanName, () -> {
1552                     // update Remote BC Group
1553                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1554                     try {
1555                         elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId);
1556                     } catch (RuntimeException e) {
1557                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1558                     }
1559                     Set<String> interfaceLists = new HashSet<>();
1560                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1561                     for (String ifName : interfaceLists) {
1562                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1563                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1564                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1565                             if (isOperational(interfaceInfo)) {
1566                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1567                             }
1568                             return Collections.emptyList();
1569                         }, ElanConstants.JOB_MAX_RETRIES);
1570                     }
1571                     return Collections.emptyList();
1572                 }, ElanConstants.JOB_MAX_RETRIES);
1573             }
1574
1575         }
1576     }
1577
1578     /**
1579      * Handle external tunnel state event.
1580      *
1581      * @param externalTunnel
1582      *            the external tunnel
1583      * @param intrf
1584      *            the interface
1585      * @throws ElanException in case of issues creating the flow objects
1586      */
1587     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1588         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1589             return;
1590         }
1591         // dpId/externalNodeId will be available either in source or destination
1592         // based on the tunnel end point
1593         BigInteger dpId = null;
1594         NodeId externalNodeId = null;
1595         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1596             dpId = new BigInteger(externalTunnel.getSourceDevice());
1597             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1598         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1599             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1600             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1601         }
1602         if (dpId == null || externalNodeId == null) {
1603             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1604             return;
1605         }
1606
1607         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1608         if (dpnInterfaceLists == null) {
1609             return;
1610         }
1611         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1612         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1613             String elanName = elanDpns.getElanInstanceName();
1614             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1615
1616             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1617             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1618                     || dpnInterfaces.getInterfaces().isEmpty()) {
1619                 continue;
1620             }
1621             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1622
1623             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpId);
1624             // install L2gwDevices local macs in dpn.
1625             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1626             // Install dpn macs on external device
1627             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1628                     externalNodeId);
1629         }
1630         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1631     }
1632
1633     /**
1634      * Installs dpn macs in external device. first it checks if the physical
1635      * locator towards this dpn tep is present or not if the physical locator is
1636      * present go ahead and add the ucast macs otherwise update the mcast mac
1637      * entry to include this dpn tep ip and schedule the job to put ucast macs
1638      * once the physical locator is programmed in device
1639      *
1640      * @param elanName
1641      *            the elan name
1642      * @param lstElanInterfaceNames
1643      *            the lst Elan interface names
1644      * @param dpnId
1645      *            the dpn id
1646      * @param externalNodeId
1647      *            the external node id
1648      */
1649     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1650             NodeId externalNodeId) {
1651         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1652                 externalNodeId.getValue());
1653         if (elanL2GwDevice == null) {
1654             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1655             return;
1656         }
1657         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1658         if (dpnTepIp == null) {
1659             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1660                     externalNodeId);
1661             return;
1662         }
1663
1664         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1665         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1666                 LogicalDatastoreType.OPERATIONAL);
1667         boolean phyLocAlreadyExists =
1668                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1669                 dpnTepIp);
1670         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1671                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1672         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1673
1674         if (phyLocAlreadyExists) {
1675             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1676             return;
1677         }
1678         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1679         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1680     }
1681
1682     /**
1683      * Validate external tunnel state event.
1684      *
1685      * @param externalTunnel
1686      *            the external tunnel
1687      * @param intrf
1688      *            the intrf
1689      * @return true, if successful
1690      */
1691     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1692         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1693             String srcDevice = externalTunnel.getDestinationDevice();
1694             String destDevice = externalTunnel.getSourceDevice();
1695             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1696                     LogicalDatastoreType.CONFIGURATION);
1697             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1698                     otherEndPointExtTunnel);
1699             if (otherEndPointExtTunnel != null) {
1700                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1701                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1702                 if (otherEndPointInterfaceOperational) {
1703                     return true;
1704                 } else {
1705                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1706                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1707                 }
1708             }
1709         }
1710         return false;
1711     }
1712
1713     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1714         List<MatchInfo> mkMatches = new ArrayList<>();
1715         // Matching metadata
1716         mkMatches.add(
1717                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1718         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1719         return mkMatches;
1720     }
1721
1722     @Override
1723     protected ElanInterfaceManager getDataTreeChangeListener() {
1724         return this;
1725     }
1726
1727     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1728                                              BigInteger dpId) {
1729         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1730         elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1731         try {
1732             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1733         } catch (InterruptedException e) {
1734             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1735         }
1736     }
1737 }