elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.collect.Lists;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.util.ArrayList;
  21 import java.util.Collections;
  22 import java.util.HashSet;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Objects;
  26 import java.util.Queue;
  27 import java.util.Set;
  28 import java.util.concurrent.ConcurrentHashMap;
  29 import java.util.concurrent.ConcurrentLinkedQueue;
  30
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34
  35 import org.apache.commons.lang3.StringUtils;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  38 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  46 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.itm.globals.ITMConstants;
  49 import org.opendaylight.genius.mdsalutil.FlowEntity;
  50 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  51 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  52 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  53 import org.opendaylight.genius.mdsalutil.MatchInfo;
  54 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  55 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  65 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  70 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  71 import org.opendaylight.netvirt.elan.ElanException;
  72 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  73 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  74 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  75 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  76 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  77 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  78 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  79 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  81 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  82 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  83 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  84 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  85 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  86 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  87 import org.opendaylight.serviceutils.srm.RecoverableListener;
  88 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  91 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 132 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.slf4j.Logger;
 135 import org.slf4j.LoggerFactory;
 136
 137 /**
 138  * Class in charge of handling creations, modifications and removals of
 139  * ElanInterfaces.
 140  *
 141  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 142  */
 143 @Singleton
 144 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 145         implements RecoverableListener {
 146     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 147     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 148     private static final boolean SH_FLAG_SET = true;
 149     private static final boolean SH_FLAG_UNSET = false;
 150
 151     private final DataBroker broker;
 152     private final ManagedNewTransactionRunner txRunner;
 153     private final IMdsalApiManager mdsalManager;
 154     private final IInterfaceManager interfaceManager;
 155     private final IdManagerService idManager;
 156     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 157     private final INeutronVpnManager neutronVpnManager;
 158     private final ElanItmUtils elanItmUtils;
 159     private final ElanEtreeUtils elanEtreeUtils;
 160     private final ElanL2GatewayUtils elanL2GatewayUtils;
 161     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 162     private final ElanUtils elanUtils;
 163     private final JobCoordinator jobCoordinator;
 164     private final ElanInstanceCache elanInstanceCache;
 165     private final ElanInterfaceCache elanInterfaceCache;
 166
 167     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 168         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 169
 170     @Inject
 171     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 172                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 173                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 174                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 175                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 176                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 177                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 178                                 final ElanInstanceCache elanInstanceCache,
 179                                 final ElanInterfaceCache elanInterfaceCache,
 180                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 181                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 182         super(ElanInterface.class, ElanInterfaceManager.class);
 183         this.broker = dataBroker;
 184         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 185         this.idManager = managerService;
 186         this.mdsalManager = mdsalApiManager;
 187         this.interfaceManager = interfaceManager;
 188         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 189         this.neutronVpnManager = neutronVpnManager;
 190         this.elanItmUtils = elanItmUtils;
 191         this.elanEtreeUtils = elanEtreeUtils;
 192         this.elanL2GatewayUtils = elanL2GatewayUtils;
 193         this.elanUtils = elanUtils;
 194         this.jobCoordinator = jobCoordinator;
 195         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 196         this.elanInstanceCache = elanInstanceCache;
 197         this.elanInterfaceCache = elanInterfaceCache;
 198         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 199     }
 200
 201     @Override
 202     @PostConstruct
 203     public void init() {
 204         registerListener();
 205     }
 206
 207     @Override
 208     public void registerListener() {
 209         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 210     }
 211
 212     @Override
 213     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 214         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 215     }
 216
 217     @Override
 218     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 219         String interfaceName = del.getName();
 220         ElanInstance elanInfo = elanInstanceCache.get(del.getElanInstanceName()).orNull();
 221         /*
 222          * Handling in case the elan instance is deleted.If the Elan instance is
 223          * deleted, there is no need to explicitly delete the elan interfaces
 224          */
 225         if (elanInfo == null) {
 226             return;
 227         }
 228         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 229         if (interfaceInfo == null && elanInfo.isExternal()) {
 230             // In deleting external network, the underlying ietf Inteface might have been removed
 231             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 232             // from Operational DS instead
 233             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 234         }
 235         String elanInstanceName = elanInfo.getElanInstanceName();
 236         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 237                 interfaceName, interfaceInfo, this);
 238         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 239     }
 240
 241     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 242     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 243             InterfaceInfo interfaceInfo) {
 244         String elanName = elanInfo.getElanInstanceName();
 245         boolean isLastElanInterface = false;
 246         boolean isLastInterfaceOnDpn = false;
 247         BigInteger dpId = null;
 248         long elanTag = elanInfo.getElanTag();
 249         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 250         WriteTransaction interfaceTx = broker.newWriteOnlyTransaction();
 251         Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 252         if (elanState == null) {
 253             interfaceTx.cancel();
 254             return Collections.emptyList();
 255         }
 256         WriteTransaction flowTx = broker.newWriteOnlyTransaction();
 257         List<String> elanInterfaces = elanState.getElanInterfaces();
 258         if (elanInterfaces.isEmpty()) {
 259             isLastElanInterface = true;
 260         }
 261         if (interfaceInfo != null) {
 262             dpId = interfaceInfo.getDpId();
 263             DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, dpId,
 264                     interfaceName, elanTag, interfaceTx);
 265             /*
 266              * If there are not elan ports, remove the unknown dmac, terminating
 267              * service table flows, remote/local bc group
 268              */
 269             if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 270                     || dpnInterfaces.getInterfaces().isEmpty()) {
 271                 // No more Elan Interfaces in this DPN
 272                 LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(), dpId);
 273                 if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 274                     removeDefaultTermFlow(dpId, elanInfo.getElanTag());
 275                 }
 276                 removeUnknownDmacFlow(dpId, elanInfo, flowTx, elanInfo.getElanTag());
 277                 removeEtreeUnknownDmacFlow(dpId, elanInfo, flowTx);
 278                 removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 279                 removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 280                 removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 281                 if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 282                     if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 283                         elanUtils.removeTerminatingServiceAction(dpId,
 284                                 elanUtils.getVxlanSegmentationId(elanInfo).intValue());
 285                     }
 286                     unsetExternalTunnelTable(dpId, elanInfo);
 287                 }
 288                 isLastInterfaceOnDpn = true;
 289             } else {
 290                 setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 291             }
 292         }
 293
 294         List<ListenableFuture<Void>> futures = new ArrayList<>();
 295         futures.add(ElanUtils.waitForTransactionToComplete(interfaceTx));
 296         futures.add(ElanUtils.waitForTransactionToComplete(flowTx));
 297
 298         if (isLastInterfaceOnDpn && dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 299             setElanAndEtreeBCGrouponOtherDpns(elanInfo, dpId);
 300         }
 301         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 302                 interfaceName, elanInfo, interfaceInfo, this, isLastElanInterface);
 303         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 304                 ElanConstants.JOB_MAX_RETRIES);
 305
 306         return futures;
 307     }
 308
 309     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 310             WriteTransaction deleteFlowGroupTx) {
 311         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 312         if (etreeLeafTag != null) {
 313             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 314             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 315         }
 316     }
 317
 318     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 319             WriteTransaction deleteFlowGroupTx) {
 320         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 321         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 322     }
 323
 324     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 325             WriteTransaction deleteFlowGroupTx) {
 326         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 327         if (etreeInstance != null) {
 328             BigInteger dpnId = interfaceInfo.getDpId();
 329             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 330             List<Bucket> listBuckets = new ArrayList<>();
 331             int bucketId = 0;
 332             listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
 333             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 334                     MDSALUtil.buildBucketLists(listBuckets));
 335             LOG.trace("deleted the localBroadCast Group:{}", group);
 336             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 337         }
 338     }
 339
 340     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 341             WriteTransaction deleteFlowGroupTx) {
 342         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 343         if (etreeInstance != null) {
 344             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 345             int bucketId = 0;
 346             int actionKey = 0;
 347             List<Bucket> listBuckets = new ArrayList<>();
 348             List<Action> listAction = new ArrayList<>();
 349             listAction.add(new ActionGroup(ElanUtils.getEtreeLeafLocalBCGId(etreeTag)).buildAction(++actionKey));
 350             listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
 351                     MDSALUtil.WATCH_GROUP));
 352             bucketId++;
 353             listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, etreeTag));
 354             BigInteger dpnId = interfaceInfo.getDpId();
 355             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 356             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
 357                     MDSALUtil.buildBucketLists(listBuckets));
 358             LOG.trace("deleting the remoteBroadCast group:{}", group);
 359             mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
 360         }
 361     }
 362
 363     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName, WriteTransaction tx) {
 364         String elanName = elanInfo.getElanInstanceName();
 365         Elan elanState = ElanUtils.getElanByName(broker, elanName);
 366         if (elanState == null) {
 367             return elanState;
 368         }
 369         List<String> elanInterfaces = elanState.getElanInterfaces();
 370         boolean isRemoved = elanInterfaces.remove(interfaceName);
 371         if (!isRemoved) {
 372             return elanState;
 373         }
 374
 375         if (elanInterfaces.isEmpty()) {
 376             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName));
 377             tx.delete(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanMacTableOperationalDataPath(elanName));
 378             tx.delete(LogicalDatastoreType.OPERATIONAL,
 379                     ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 380         } else {
 381             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 382                     .withKey(new ElanKey(elanName)).build();
 383             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanName),
 384                     updateElanState);
 385         }
 386         return elanState;
 387     }
 388
 389     private void deleteElanInterfaceFromConfigDS(String interfaceName, WriteTransaction tx) {
 390         // removing the ElanInterface from the config data_store if interface is
 391         // not present in Interface config DS
 392         if (interfaceManager.getInterfaceInfoFromConfigDataStore(interfaceName) == null
 393                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 394             tx.delete(LogicalDatastoreType.CONFIGURATION,
 395                     ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 396         }
 397     }
 398
 399     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 400             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 401         String elanName = elanInfo.getElanInstanceName();
 402         List<ListenableFuture<Void>> futures = new ArrayList<>();
 403         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 404             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(interfaceTx -> {
 405                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 406                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 407                 Optional<ElanInterfaceMac> existingElanInterfaceMac =
 408                         interfaceTx.read(LogicalDatastoreType.OPERATIONAL, elanInterfaceId).checkedGet();
 409                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 410                 if (interfaceInfo != null) {
 411                     if (existingElanInterfaceMac.isPresent()) {
 412                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 413                         if (existingMacEntries != null) {
 414                             List<PhysAddress> macAddresses = new ArrayList<>();
 415                             for (MacEntry macEntry : existingMacEntries) {
 416                                 PhysAddress macAddress = macEntry.getMacAddress();
 417                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 418                                         macAddress.getValue(), interfaceName);
 419                                 Optional<MacEntry> macEntryOptional =
 420                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 421                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 422                                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 423                                             ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 424                                 }
 425                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 426                                 macAddresses.add(macAddress);
 427                             }
 428
 429                             // Removing all those MACs from External Devices belonging
 430                             // to this ELAN
 431                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 432                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 433                             }
 434                         }
 435                     }
 436                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 437                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 438                 } else if (existingElanInterfaceMac.isPresent()) {
 439                     // Interface does not exist in ConfigDS, so lets remove everything
 440                     // about that interface related to Elan
 441                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 442                     if (macEntries != null) {
 443                         for (MacEntry macEntry : macEntries) {
 444                             PhysAddress macAddress = macEntry.getMacAddress();
 445                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 446                                 interfaceTx.delete(LogicalDatastoreType.OPERATIONAL,
 447                                         ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 448                             }
 449                         }
 450                     }
 451                 }
 452                 if (existingElanInterfaceMac.isPresent()) {
 453                     interfaceTx.delete(LogicalDatastoreType.OPERATIONAL, elanInterfaceId);
 454                 }
 455                 unbindService(interfaceName, interfaceTx);
 456                 deleteElanInterfaceFromConfigDS(interfaceName, interfaceTx);
 457             }));
 458         }));
 459         return futures;
 460     }
 461
 462     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 463                                                                          String interfaceName, long elanTag,
 464                                                                          WriteTransaction tx) {
 465         synchronized (elanName.intern()) {
 466
 467             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 468             if (dpnInterfaces != null) {
 469                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 470                 if (interfaceLists != null) {
 471                     interfaceLists.remove(interfaceName);
 472                 }
 473
 474                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 475                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 476                     deleteElanDpnInterface(elanName, dpId, tx);
 477                 } else {
 478                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 479                 }
 480             }
 481             return dpnInterfaces;
 482         }
 483     }
 484
 485     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 486         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 487         for (DpnInterfaces dpnInterface : dpnInterfaces) {
 488             BigInteger currentDpId = dpnInterface.getDpId();
 489             if (!currentDpId.equals(dpId)) {
 490                 for (String elanInterface : dpnInterface.getInterfaces()) {
 491                     ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 492                     if (macs == null || macs.getMacEntry() == null) {
 493                         continue;
 494                     }
 495                     for (MacEntry mac : macs.getMacEntry()) {
 496                         removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 497                         removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac);
 498                     }
 499                 }
 500             }
 501         }
 502     }
 503
 504     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 505         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 506         if (etreeLeafTag != null) {
 507             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac);
 508         }
 509     }
 510
 511     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac) {
 512         mdsalManager
 513                 .removeFlow(dpId,
 514                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 515                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 516                                         mac.getMacAddress().getValue(), elanTag)));
 517     }
 518
 519     /*
 520     * Possible Scenarios for update
 521     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 522         then all {1,2,3,4} should be removed
 523
 524         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 525         then all {1,2,3,4} should be added
 526
 527         c. if orig = {1,2,3,4} updated={2,3,4}
 528         then 1 should be removed
 529
 530         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 531         then we should just add 5
 532
 533         e. if orig = {1,2,3,4} updated={2,3,4,5}
 534         then 1 should be removed , 5 should be added
 535     * */
 536     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 537     @Override
 538     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 539         // updating the static-Mac Entries for the existing elanInterface
 540         String elanName = update.getElanInstanceName();
 541         String interfaceName = update.getName();
 542
 543         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 544         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 545         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 546         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 547
 548         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 549                 deletedEntry.getMacAddress()));
 550
 551         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 552         * Hence add the macentries for the same.*/
 553         for (StaticMacEntries staticMacEntry : updatedEntries) {
 554             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 555                     staticMacEntry.getMacAddress());
 556             Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 557                     LogicalDatastoreType.OPERATIONAL, macEntryIdentifier);
 558             WriteTransaction tx = broker.newWriteOnlyTransaction();
 559             if (existingMacEntry.isPresent()) {
 560                 elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 561                         elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 562                         tx);
 563             } else {
 564                 elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 565                         elanName, interfaceName, staticMacEntry, tx);
 566             }
 567             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(tx), LOG,
 568                     "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 569         }
 570     }
 571
 572     @Override
 573     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 574         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 575             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 576             String interfaceName = elanInterfaceAdded.getName();
 577             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 578             if (interfaceInfo == null) {
 579                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 580                 return;
 581             }
 582             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 583
 584             if (elanInstance == null) {
 585                 elanInstance = new ElanInstanceBuilder().setElanInstanceName(elanInstanceName)
 586                         .setDescription(elanInterfaceAdded.getDescription()).build();
 587                 // Add the ElanInstance in the Configuration data-store
 588                 List<String> elanInterfaces = new ArrayList<>();
 589                 elanInterfaces.add(interfaceName);
 590                 elanInstance = ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 591             }
 592
 593             Long elanTag = elanInstance.getElanTag();
 594             // If elan tag is not updated, then put the elan interface into
 595             // unprocessed entry map and entry. Let entries
 596             // in this map get processed during ELAN update DCN.
 597             if (elanTag == null) {
 598                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 599                 if (elanInterfaces == null) {
 600                     elanInterfaces = new ConcurrentLinkedQueue<>();
 601                 }
 602                 elanInterfaces.add(elanInterfaceAdded);
 603                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 604                 return;
 605             }
 606             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 607                     interfaceInfo, elanInstance, this);
 608             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 609         }), LOG, "Error procedding added ELAN interface");
 610     }
 611
 612     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) throws ElanException {
 613         List<ListenableFuture<Void>> futures = new ArrayList<>();
 614         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 615         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 616             return futures;
 617         }
 618         for (ElanInterface elanInterface : elanInterfaces) {
 619             String interfaceName = elanInterface.getName();
 620             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 621             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 622         }
 623         return futures;
 624     }
 625
 626     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 627             WriteTransaction writeFlowGroupTx) throws ElanException {
 628         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 629                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 630         List<DpnInterfaces> dpnInterfaceLists = null;
 631         if (elanDpnInterfacesList != null) {
 632             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 633         }
 634         if (dpnInterfaceLists == null) {
 635             dpnInterfaceLists = new ArrayList<>();
 636         }
 637         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 638             BigInteger dstDpId = interfaceInfo.getDpId();
 639             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 640                 continue;
 641             }
 642             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 643             for (String remoteIf : remoteElanInterfaces) {
 644                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 645                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 646                 if (elanIfMac == null || remoteInterface == null) {
 647                     continue;
 648                 }
 649                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 650                 if (remoteMacEntries != null) {
 651                     for (MacEntry macEntry : remoteMacEntries) {
 652                         String macAddress = macEntry.getMacAddress().getValue();
 653                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 654                                 dstDpId, elanInstance.getElanInstanceName());
 655                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 656                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 657                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 658                     }
 659                 }
 660             }
 661         }
 662     }
 663
 664     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 665     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 666             InterfaceInfo interfaceInfo, ElanInstance elanInstance) throws ElanException {
 667         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 668         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 669         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 670
 671         String interfaceName = elanInterface.getName();
 672         String elanInstanceName = elanInterface.getElanInstanceName();
 673
 674         Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 675         WriteTransaction tx = broker.newWriteOnlyTransaction();
 676         if (elanInfo == null) {
 677             List<String> elanInterfaces = new ArrayList<>();
 678             elanInterfaces.add(interfaceName);
 679             ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, tx);
 680         } else {
 681             createElanStateList(elanInstanceName, interfaceName, tx);
 682         }
 683         boolean isFirstInterfaceInDpn = false;
 684         // Specific actions to the DPN where the ElanInterface has been added,
 685         // for example, programming the
 686         // External tunnel table if needed or adding the ElanInterface to the
 687         // DpnInterfaces in the operational DS.
 688         BigInteger dpId = interfaceInfo.getDpId();
 689         DpnInterfaces dpnInterfaces = null;
 690         if (dpId != null && !dpId.equals(ElanConstants.INVALID_DPN)) {
 691             synchronized (elanInstanceName.intern()) {
 692                 InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 693                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
 694                 Optional<DpnInterfaces> existingElanDpnInterfaces = ElanUtils.read(broker,
 695                         LogicalDatastoreType.OPERATIONAL, elanDpnInterfaces);
 696                 if (ElanUtils.isVlan(elanInstance)) {
 697                     isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 698                             elanInstanceName, existingElanDpnInterfaces);
 699                 } else {
 700                     isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 701                 }
 702                 if (isFirstInterfaceInDpn) {
 703                     // ELAN's 1st ElanInterface added to this DPN
 704                     if (!existingElanDpnInterfaces.isPresent()) {
 705                         dpnInterfaces = createElanInterfacesList(elanInstanceName, interfaceName, dpId, tx);
 706                     } else {
 707                         List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 708                         elanInterfaces.add(interfaceName);
 709                         dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId,
 710                                 elanInterfaces, tx);
 711                     }
 712                     // The 1st ElanInterface in a DPN must program the Ext Tunnel
 713                     // table, but only if Elan has VNI
 714                     if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 715                         setExternalTunnelTable(dpId, elanInstance);
 716                     }
 717                     elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 718                 } else {
 719                     List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 720                     elanInterfaces.add(interfaceName);
 721                     if (elanInterfaces.size() == 1) { // 1st dpn interface
 722                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(dpId, elanInstance, interfaceName);
 723                     }
 724                     dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, dpId, elanInterfaces, tx);
 725                 }
 726             }
 727         }
 728
 729         // add code to install Local/Remote BC group, unknow DMAC entry,
 730         // terminating service table flow entry
 731         // call bindservice of interfacemanager to create ingress table flow
 732         // enty.
 733         // Add interface to the ElanInterfaceForwardingEntires Container
 734         createElanInterfaceTablesList(interfaceName, tx);
 735         List<ListenableFuture<Void>> futures = new ArrayList<>();
 736         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 737         installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, dpnInterfaces, isFirstInterfaceInDpn);
 738
 739         // add the vlan provider interface to remote BC group for the elan
 740         // for internal vlan networks
 741         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 742             if (interfaceManager.isExternalInterface(interfaceName)) {
 743                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 744                 handleExternalInterfaceEvent(elanInstance, dpnInterfaces, dpId);
 745             }
 746         }
 747
 748         if (isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 749             //update the remote-DPNs remoteBC group entry with Tunnels
 750             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance, dpId);
 751             setElanAndEtreeBCGrouponOtherDpns(elanInstance, dpId);
 752         }
 753
 754         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 755         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 756                 elanInterface, interfaceInfo, elanInstance, isFirstInterfaceInDpn, this);
 757         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 758         return futures;
 759     }
 760
 761     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 762     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 763             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn)
 764             throws ElanException {
 765         String elanInstanceName = elanInstance.getElanInstanceName();
 766         String interfaceName = elanInterface.getName();
 767         WriteTransaction tx = broker.newWriteOnlyTransaction();
 768         BigInteger dpId = interfaceInfo.getDpId();
 769         WriteTransaction writeFlowGroupTx = broker.newWriteOnlyTransaction();
 770         installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 771                 isFirstInterfaceInDpn, tx, writeFlowGroupTx);
 772
 773         List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 774         List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 775
 776         boolean isInterfaceOperational = isOperational(interfaceInfo);
 777         if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 778             for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 779                 InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 780                         staticMacEntry.getMacAddress());
 781                 Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 782                         LogicalDatastoreType.OPERATIONAL, macId);
 783                 if (existingMacEntry.isPresent()) {
 784                     elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 785                             elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 786                             existingMacEntry.get(), tx);
 787                 } else {
 788                     elanForwardingEntriesHandler
 789                             .addElanInterfaceForwardingTableList(elanInstanceName, interfaceName, staticMacEntry, tx);
 790                 }
 791
 792                 if (isInterfaceOperational) {
 793                     // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 794                     // DPNs
 795                     String macAddress = staticMacEntry.getMacAddress().getValue();
 796                     LOG.info("programming smac and dmacs for {} on source and other DPNs for elan {} and interface {}",
 797                             macAddress, elanInstanceName, interfaceName);
 798                     elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 799                             staticMacEntry.getMacAddress().getValue(), true, writeFlowGroupTx);
 800                 }
 801             }
 802
 803             if (isInterfaceOperational) {
 804                 // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 805                 // on purpose.
 806                 for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 807                     staticMacAddresses.add(staticMacEntry.getMacAddress());
 808                 }
 809                 elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 810                         staticMacAddresses);
 811             }
 812         }
 813         List<ListenableFuture<Void>> futures = new ArrayList<>();
 814         futures.add(ElanUtils.waitForTransactionToComplete(tx));
 815         futures.add(ElanUtils.waitForTransactionToComplete(writeFlowGroupTx));
 816         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 817             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 818             try {
 819                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 820                 if (neutronPort != null) {
 821                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 822                 }
 823             } catch (IllegalArgumentException ex) {
 824                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 825             }
 826         }
 827         return futures;
 828     }
 829
 830     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 831             PhysAddress physAddress) {
 832         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 833         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 834         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 835                 LogicalDatastoreType.OPERATIONAL, macId);
 836
 837         if (!existingMacEntry.isPresent()) {
 838             return;
 839         }
 840
 841         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 842                 .withKey(new MacEntryKey(physAddress)).build();
 843         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 844                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 845     }
 846
 847     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 848             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 849         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 850         if (!existingElanDpnInterfaces.isPresent()) {
 851             return true;
 852         }
 853         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 854             return false;
 855         }
 856         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 857         int dummyInterfaceCount =  0;
 858         if (dpnInterfaces.getInterfaces().contains(routerPortUuid)) {
 859             dummyInterfaceCount++;
 860         }
 861         if (dpnInterfaces.getInterfaces().contains(elanInstanceName)) {
 862             dummyInterfaceCount++;
 863         }
 864         if (dpnInterfaces.getInterfaces().size() - dummyInterfaceCount == 0) {
 865             return true;
 866         }
 867         return false;
 868     }
 869
 870     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 871         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 872                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 873     }
 874
 875     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 876             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, WriteTransaction tx,
 877             WriteTransaction writeFlowGroupTx) throws ElanException {
 878         if (!isOperational(interfaceInfo)) {
 879             return;
 880         }
 881         BigInteger dpId = interfaceInfo.getDpId();
 882         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 883             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, writeFlowGroupTx);
 884         }
 885         setupFilterEqualsTable(elanInstance, interfaceInfo, writeFlowGroupTx);
 886         if (isFirstInterfaceInDpn) {
 887             // Terminating Service , UnknownDMAC Table.
 888             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 889             // for ELAN Instance is VxLAN
 890             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 891                 setupTerminateServiceTable(elanInstance, dpId, writeFlowGroupTx);
 892             }
 893             setupUnknownDMacTable(elanInstance, dpId, writeFlowGroupTx);
 894             /*
 895              * Install remote DMAC flow. This is required since this DPN is
 896              * added later to the elan instance and remote DMACs of other
 897              * interfaces in this elan instance are not present in the current
 898              * dpn.
 899              */
 900             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 901                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 902                         elanInstance.getElanInstanceName());
 903                 programRemoteDmacFlow(elanInstance, interfaceInfo, writeFlowGroupTx);
 904             }
 905         }
 906         // bind the Elan service to the Interface
 907         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), tx);
 908     }
 909
 910     public void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 911             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn) {
 912         if (!isOperational(interfaceInfo)) {
 913             return;
 914         }
 915         // LocalBroadcast Group creation with elan-Interfaces
 916         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo);
 917         if (isFirstInterfaceInDpn) {
 918             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 919             BigInteger dpId = interfaceInfo.getDpId();
 920             // RemoteBroadcast Group creation
 921             try {
 922                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 923             } catch (InterruptedException e1) {
 924                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 925             }
 926             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId);
 927             try {
 928                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 929             } catch (InterruptedException e1) {
 930                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 931             }
 932         }
 933     }
 934
 935     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 936             WriteTransaction writeFlowGroupTx) {
 937         int ifTag = interfaceInfo.getInterfaceTag();
 938         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 939                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 940                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 941                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 942                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 943
 944         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flow, writeFlowGroupTx);
 945
 946         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 947                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 948                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 949                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 950
 951         mdsalManager.addFlowToTx(interfaceInfo.getDpId(), flowEntry, writeFlowGroupTx);
 952     }
 953
 954     // TODO skitt Fix the exception handling here
 955     @SuppressWarnings("checkstyle:IllegalCatch")
 956     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 957     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 958             TypedReadWriteTransaction<Configuration> flowTx) {
 959         try {
 960             int ifTag = interfaceInfo.getInterfaceTag();
 961             Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 962                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
 963
 964             mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
 965
 966             Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 967                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 968                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 969                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 970
 971             mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
 972         } catch (Exception e) {
 973             LOG.error("Error removing flow", e);
 974             throw new RuntimeException("Error removing flow", e);
 975         }
 976     }
 977
 978     private List<Bucket> getRemoteBCGroupBucketInfos(ElanInstance elanInfo, int bucketKeyStart,
 979                                                      InterfaceInfo interfaceInfo, long elanTag) {
 980         return elanL2GatewayMulticastUtils.getRemoteBCGroupBuckets(elanInfo, null, interfaceInfo.getDpId(),
 981                 bucketKeyStart, elanTag);
 982     }
 983
 984     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId) {
 985         int elanTag = elanInfo.getElanTag().intValue();
 986         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 987         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId);
 988         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 989         if (etreeInstance != null) {
 990             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
 991             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
 992             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId);
 993         }
 994     }
 995
 996     @SuppressWarnings("checkstyle:IllegalCatch")
 997     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId) {
 998         int bucketId = 0;
 999         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1000         if (elanDpns != null) {
1001             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1002             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1003                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1004                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1005                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1006                     List<Action> listAction = new ArrayList<>();
1007                     int actionKey = 0;
1008                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1009                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1010                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1011                     bucketId++;
1012                     for (DpnInterfaces otherFes : dpnInterfaces) {
1013                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1014                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1015                             && !otherFes.getInterfaces().isEmpty()) {
1016                             try {
1017                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1018                                         dpnInterface.getDpId(), otherFes.getDpId(),
1019                                         elanUtils.isOpenstackVniSemanticsEnforced()
1020                                                 ? elanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1021                                 if (!remoteListActionInfo.isEmpty()) {
1022                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1023                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1024                                     bucketId++;
1025                                 }
1026                             } catch (Exception ex) {
1027                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1028                                         + "Logical Group Interface not found between source Dpn - {}, "
1029                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1030                                 return;
1031                             }
1032                         }
1033                     }
1034                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1035                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1036                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1037
1038                     if (remoteListBucketInfo.isEmpty()) {
1039                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1040                         continue;
1041                     }
1042                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1043                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1044                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1045                     mdsalManager.syncInstallGroup(dpnInterface.getDpId(), group);
1046                 }
1047             }
1048             try {
1049                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1050             } catch (InterruptedException e1) {
1051                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1052             }
1053         }
1054     }
1055
1056     /**
1057      * Returns the bucket info with the given interface as the only bucket.
1058      */
1059     private Bucket getLocalBCGroupBucketInfo(InterfaceInfo interfaceInfo, int bucketIdStart) {
1060         return MDSALUtil.buildBucket(getInterfacePortActions(interfaceInfo), MDSALUtil.GROUP_WEIGHT, bucketIdStart,
1061                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1062     }
1063
1064     private List<MatchInfo> buildMatchesForVni(Long vni) {
1065         List<MatchInfo> mkMatches = new ArrayList<>();
1066         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1067         mkMatches.add(match);
1068         return mkMatches;
1069     }
1070
1071     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1072         List<InstructionInfo> mkInstructions = new ArrayList<>();
1073         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1074         return mkInstructions;
1075     }
1076
1077     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1078         List<MatchInfo> mkMatches = new ArrayList<>();
1079         // Matching metadata
1080         mkMatches.add(new MatchMetadata(
1081                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1082         return mkMatches;
1083     }
1084
1085     /**
1086      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1087      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1088      *
1089      * @param elanTag
1090      *            elanTag to be written in metadata when flow is selected
1091      * @return the instructions ready to be installed in a flow
1092      */
1093     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1094         List<InstructionInfo> mkInstructions = new ArrayList<>();
1095         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1096                 .getElanMetadataMask()));
1097         /* applicable for EXTERNAL_TUNNEL_TABLE only
1098         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1099         */
1100         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1101         return mkInstructions;
1102     }
1103
1104     // Install DMAC entry on dst DPN
1105     @SuppressWarnings("checkstyle:ForbidCertainMethod")
1106     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1107             BigInteger dstDpId) throws ElanException {
1108         String interfaceName = interfaceInfo.getInterfaceName();
1109         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1110         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1111             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1112             WriteTransaction writeFlowTx = broker.newWriteOnlyTransaction();
1113             for (MacEntry macEntry : macEntries) {
1114                 String macAddress = macEntry.getMacAddress().getValue();
1115                 LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress, interfaceName);
1116                 synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1117                         interfaceInfo.getDpId())) {
1118                     LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation", macAddress);
1119                     elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress,
1120                             writeFlowTx);
1121                 }
1122             }
1123             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(writeFlowTx));
1124         }
1125         return Collections.emptyList();
1126     }
1127
1128     private void createDropBucket(List<Bucket> listBucket) {
1129         List<Action> actionsInfos = new ArrayList<>();
1130         actionsInfos.add(new ActionDrop().buildAction());
1131         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1132                 MDSALUtil.WATCH_GROUP);
1133         listBucket.add(dropBucket);
1134     }
1135
1136     public void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1137             InterfaceInfo interfaceInfo) {
1138         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1139         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo);
1140     }
1141
1142     public void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1143             InterfaceInfo interfaceInfo) {
1144         List<Bucket> listBucket = new ArrayList<>();
1145         int bucketId = 0;
1146         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1147
1148         List<String> interfaces = new ArrayList<>();
1149         if (newDpnInterface != null) {
1150             interfaces = newDpnInterface.getInterfaces();
1151         }
1152         for (String ifName : interfaces) {
1153             // In case if there is a InterfacePort in the cache which is not in
1154             // operational state, skip processing it
1155             InterfaceInfo ifInfo = interfaceManager
1156                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1157             if (!isOperational(ifInfo)) {
1158                 continue;
1159             }
1160
1161             if (!interfaceManager.isExternalInterface(ifName)) {
1162                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1163                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1164                 bucketId++;
1165             }
1166         }
1167
1168         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1169                 MDSALUtil.buildBucketLists(listBucket));
1170         LOG.trace("installing the localBroadCast Group:{}", group);
1171         mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1172     }
1173
1174     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1175             InterfaceInfo interfaceInfo) {
1176         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1177         if (etreeInstance != null) {
1178             List<Bucket> listBucket = new ArrayList<>();
1179             int bucketId = 0;
1180
1181             List<String> interfaces = new ArrayList<>();
1182             if (newDpnInterface != null) {
1183                 interfaces = newDpnInterface.getInterfaces();
1184             }
1185             for (String ifName : interfaces) {
1186                 // In case if there is a InterfacePort in the cache which is not
1187                 // in
1188                 // operational state, skip processing it
1189                 InterfaceInfo ifInfo = interfaceManager
1190                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1191                 if (!isOperational(ifInfo)) {
1192                     continue;
1193                 }
1194
1195                 if (!interfaceManager.isExternalInterface(ifName)) {
1196                     // only add root interfaces
1197                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1198                 }
1199             }
1200
1201             if (listBucket.isEmpty()) { // No Buckets
1202                 createDropBucket(listBucket);
1203             }
1204
1205             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1206             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1207             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1208                     MDSALUtil.buildBucketLists(listBucket));
1209             LOG.trace("installing the localBroadCast Group:{}", group);
1210             mdsalManager.syncInstallGroup(interfaceInfo.getDpId(), group);
1211         }
1212     }
1213
1214     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1215             InterfaceInfo ifInfo) {
1216         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1217         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1218             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1219                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1220             bucketId++;
1221         }
1222         return bucketId;
1223     }
1224
1225     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1226             WriteTransaction deleteFlowGroupTx) {
1227         BigInteger dpnId = interfaceInfo.getDpId();
1228         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1229         List<Bucket> listBuckets = new ArrayList<>();
1230         int bucketId = 0;
1231         listBuckets.add(getLocalBCGroupBucketInfo(interfaceInfo, bucketId));
1232         // listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, 1,
1233         // interfaceInfo));
1234         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1235                 MDSALUtil.buildBucketLists(listBuckets));
1236         LOG.trace("deleted the localBroadCast Group:{}", group);
1237         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1238     }
1239
1240     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1241             WriteTransaction deleteFlowGroupTx) {
1242         int bucketId = 0;
1243         int actionKey = 0;
1244         Long elanTag = elanInfo.getElanTag();
1245         List<Bucket> listBuckets = new ArrayList<>();
1246         List<Action> listAction = new ArrayList<>();
1247         listAction.add(new ActionGroup(++actionKey, ElanUtils.getElanLocalBCGId(elanTag)).buildAction());
1248         listBuckets.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT,
1249                 MDSALUtil.WATCH_GROUP));
1250         bucketId++;
1251         listBuckets.addAll(getRemoteBCGroupBucketInfos(elanInfo, bucketId, interfaceInfo, elanTag));
1252         BigInteger dpnId = interfaceInfo.getDpId();
1253         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1254         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1255                 MDSALUtil.buildBucketLists(listBuckets));
1256         LOG.trace("deleting the remoteBroadCast group:{}", group);
1257         mdsalManager.removeGroupToTx(dpnId, group, deleteFlowGroupTx);
1258     }
1259
1260     /**
1261      * Installs a flow in the External Tunnel table consisting in translating
1262      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1263      * elanTag that will be used later in the ELANs pipeline.
1264      *
1265      * @param dpnId
1266      *            the dpn id
1267      * @param elanInfo
1268      *            the elan info
1269      */
1270     public void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1271         long elanTag = elanInfo.getElanTag();
1272         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1273                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1274                 elanInfo.getElanInstanceName(), // flowName
1275                 0, // idleTimeout
1276                 0, // hardTimeout
1277                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1278                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1279                 getInstructionsIntOrExtTunnelTable(elanTag));
1280
1281         mdsalManager.installFlow(flowEntity);
1282     }
1283
1284     /**
1285      * Removes, from External Tunnel table, the flow that translates from VNI to
1286      * elanTag. Important: ensure this method is only called whenever there is
1287      * no other ElanInterface in the specified DPN
1288      *
1289      * @param dpnId
1290      *            DPN whose Ext Tunnel table is going to be modified
1291      * @param elanInfo
1292      *            holds the elanTag needed for selecting the flow to be removed
1293      */
1294     public void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo) {
1295         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1296         // last ElanInstance plus
1297         // adding a new one does (almost at the same time) are executed in that
1298         // exact order
1299
1300         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1301         FlowEntity flowEntity = new FlowEntityBuilder()
1302             .setDpnId(dpnId)
1303             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1304             .setFlowId(flowId)
1305             .build();
1306         mdsalManager.removeFlow(flowEntity);
1307     }
1308
1309     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1310         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1311         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1312     }
1313
1314     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1315             WriteTransaction writeFlowGroupTx) {
1316         List<? extends MatchInfoBase> listMatchInfoBase;
1317         List<InstructionInfo> instructionInfos;
1318         long serviceId;
1319         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1320             serviceId = elanTag;
1321             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1322             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1323         } else {
1324             serviceId = elanUtils.getVxlanSegmentationId(elanInfo);
1325             listMatchInfoBase = buildMatchesForVni(serviceId);
1326             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1327         }
1328         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1329                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1330                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1331                 instructionInfos);
1332         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1333     }
1334
1335     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1336             WriteTransaction writeFlowGroupTx) {
1337         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1338         if (etreeInstance != null) {
1339             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1340         }
1341     }
1342
1343     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, WriteTransaction writeFlowGroupTx) {
1344         long elanTag = elanInfo.getElanTag();
1345         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1346         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1347         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1348     }
1349
1350     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1351             WriteTransaction writeFlowGroupTx) {
1352         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1353         if (etreeLeafTag != null) {
1354             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1355             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1356             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1357         }
1358     }
1359
1360     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1361             WriteTransaction writeFlowGroupTx) {
1362         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1363                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1364                 5, elanInfo.getElanInstanceName(), 0, 0,
1365                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1366                 getMatchesForElanTag(elanTag, /* SH flag */false),
1367                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1368
1369         mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1370     }
1371
1372     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1373             WriteTransaction writeFlowGroupTx) {
1374         // only if ELAN can connect to external network, perform the following
1375
1376         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1377             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1378                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1379                     5, elanInfo.getElanInstanceName(), 0, 0,
1380                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1381                     getMatchesForElanTag(elanTag, /* SH flag */true),
1382                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1383             mdsalManager.addFlowToTx(flowEntity, writeFlowGroupTx);
1384         }
1385     }
1386
1387
1388     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo, WriteTransaction deleteFlowGroupTx,
1389             long elanTag) {
1390         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1391                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1392         mdsalManager.removeFlowToTx(dpId, flow, deleteFlowGroupTx);
1393
1394         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1395             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1396                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1397                     .build();
1398             mdsalManager.removeFlowToTx(dpId, flow2, deleteFlowGroupTx);
1399         }
1400     }
1401
1402     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1403         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1404     }
1405
1406     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag, WriteTransaction tx) {
1407         if (isStandardElanService(elanInterface)) {
1408             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1409                     elanInterface.getName(), lportTag, tx);
1410         } else { // Etree service
1411             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1412         }
1413     }
1414
1415     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1416             WriteTransaction tx) {
1417         int instructionKey = 0;
1418         List<Instruction> instructions = new ArrayList<>();
1419         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1420                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1421
1422         List<Action> actions = new ArrayList<>();
1423         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1424                 lportTag).buildAction());
1425         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1426                 elanTag).buildAction());
1427         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1428
1429         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1430                 ++instructionKey));
1431
1432         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1433         BoundServices serviceInfo = ElanUtils.getBoundServices(
1434                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1435                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1436         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1437         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1438                 bindServiceId);
1439         if (!existingElanService.isPresent()) {
1440             tx.put(LogicalDatastoreType.CONFIGURATION, bindServiceId, serviceInfo,
1441                     WriteTransaction.CREATE_MISSING_PARENTS);
1442         }
1443     }
1444
1445     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1446             WriteTransaction tx) {
1447         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1448             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1449                     lportTag, tx);
1450         } else {
1451             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1452             if (etreeInstance == null) {
1453                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1454                         elanInterface.getName(), elanInfo.getElanInstanceName());
1455             } else {
1456                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1457                         elanInterface.getName(), lportTag, tx);
1458             }
1459         }
1460     }
1461
1462     private boolean isStandardElanService(ElanInterface elanInterface) {
1463         return elanInterface.augmentation(EtreeInterface.class) == null;
1464     }
1465
1466     protected void unbindService(String interfaceName, ReadWriteTransaction tx) throws ReadFailedException {
1467         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1468         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1469         if (tx.read(LogicalDatastoreType.CONFIGURATION, bindServiceId).checkedGet().isPresent()) {
1470             tx.delete(LogicalDatastoreType.CONFIGURATION, bindServiceId);
1471         }
1472     }
1473
1474     private String getFlowRef(long tableId, long elanTag) {
1475         return String.valueOf(tableId) + elanTag;
1476     }
1477
1478     private String getFlowRef(long tableId, long elanTag, String flowName) {
1479         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1480                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1481     }
1482
1483     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1484         return String.valueOf(tableId) + elanTag + shFlag;
1485     }
1486
1487     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1488         List<Action> listAction = new ArrayList<>();
1489         int actionKey = 0;
1490         listAction.add(
1491             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1492         actionKey++;
1493         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1494         return listAction;
1495     }
1496
1497     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1498             List<String> interfaceNames, WriteTransaction tx) {
1499         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1500                 .withKey(new DpnInterfacesKey(dpId)).build();
1501         tx.put(LogicalDatastoreType.OPERATIONAL,
1502                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1503                 WriteTransaction.CREATE_MISSING_PARENTS);
1504         return dpnInterface;
1505     }
1506
1507     /**
1508      * Delete elan dpn interface from operational DS.
1509      *
1510      * @param elanInstanceName
1511      *            the elan instance name
1512      * @param dpId
1513      *            the dp id
1514      */
1515     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId, WriteTransaction tx) {
1516         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1517                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1518         Optional<DpnInterfaces> dpnInterfaces = ElanUtils.read(broker,
1519                 LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1520         if (dpnInterfaces.isPresent()) {
1521             tx.delete(LogicalDatastoreType.OPERATIONAL, dpnInterfacesId);
1522         }
1523     }
1524
1525     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1526             WriteTransaction tx) {
1527         List<String> interfaceNames = new ArrayList<>();
1528         interfaceNames.add(interfaceName);
1529         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1530                 .withKey(new DpnInterfacesKey(dpId)).build();
1531         tx.put(LogicalDatastoreType.OPERATIONAL,
1532                 ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1533                 WriteTransaction.CREATE_MISSING_PARENTS);
1534         return dpnInterface;
1535     }
1536
1537     private void createElanInterfaceTablesList(String interfaceName, WriteTransaction tx) {
1538         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1539                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1540         Optional<ElanInterfaceMac> interfaceMacTables = ElanUtils.read(broker,
1541                 LogicalDatastoreType.OPERATIONAL, elanInterfaceMacTables);
1542         // Adding new Elan Interface Port to the operational DataStore without
1543         // Static-Mac Entries..
1544         if (!interfaceMacTables.isPresent()) {
1545             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1546                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1547             tx.put(LogicalDatastoreType.OPERATIONAL,
1548                     ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1549                     WriteTransaction.CREATE_MISSING_PARENTS);
1550         }
1551     }
1552
1553     private void createElanStateList(String elanInstanceName, String interfaceName, WriteTransaction tx) {
1554         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1555         Optional<Elan> elanInterfaceLists = ElanUtils.read(broker,
1556                 LogicalDatastoreType.OPERATIONAL, elanInstance);
1557         // Adding new Elan Interface Port to the operational DataStore without
1558         // Static-Mac Entries..
1559         if (elanInterfaceLists.isPresent()) {
1560             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1561             if (interfaceLists == null) {
1562                 interfaceLists = new ArrayList<>();
1563             }
1564             interfaceLists.add(interfaceName);
1565             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1566                     .withKey(new ElanKey(elanInstanceName)).build();
1567             tx.put(LogicalDatastoreType.OPERATIONAL, ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName),
1568                     elanState, WriteTransaction.CREATE_MISSING_PARENTS);
1569         }
1570     }
1571
1572     private boolean isOperational(InterfaceInfo interfaceInfo) {
1573         if (interfaceInfo == null) {
1574             return false;
1575         }
1576         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1577     }
1578
1579     @SuppressWarnings("checkstyle:IllegalCatch")
1580     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1581         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1582         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1583                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1584         if (dpnInterfaceLists == null) {
1585             return;
1586         }
1587         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1588         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1589             int cnt = 0;
1590             String elanName = elanDpns.getElanInstanceName();
1591             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1592             if (elanInfo == null) {
1593                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1594                         + "skipping this ELAN for tunnel handling", elanName);
1595                 continue;
1596             }
1597             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1598                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1599                 continue;
1600             }
1601             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1602             if (dpnInterfaces == null) {
1603                 continue;
1604             }
1605             DpnInterfaces dstDpnIf = null;
1606             for (DpnInterfaces dpnIf : dpnInterfaces) {
1607                 BigInteger dpnIfDpId = dpnIf.getDpId();
1608                 if (dpnIfDpId.equals(srcDpId)) {
1609                     cnt++;
1610                 } else if (dpnIfDpId.equals(dstDpId)) {
1611                     cnt++;
1612                     dstDpnIf = dpnIf;
1613                 }
1614             }
1615             if (cnt == 2) {
1616                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1617                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1618                 jobCoordinator.enqueueJob(elanName, () -> {
1619                     // update Remote BC Group
1620                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1621                     try {
1622                         elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId);
1623                     } catch (RuntimeException e) {
1624                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1625                     }
1626                     Set<String> interfaceLists = new HashSet<>();
1627                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1628                     for (String ifName : interfaceLists) {
1629                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1630                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1631                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1632                             if (isOperational(interfaceInfo)) {
1633                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1634                             }
1635                             return Collections.emptyList();
1636                         }, ElanConstants.JOB_MAX_RETRIES);
1637                     }
1638                     return Collections.emptyList();
1639                 }, ElanConstants.JOB_MAX_RETRIES);
1640             }
1641
1642         }
1643     }
1644
1645     /**
1646      * Handle external tunnel state event.
1647      *
1648      * @param externalTunnel
1649      *            the external tunnel
1650      * @param intrf
1651      *            the interface
1652      * @throws ElanException in case of issues creating the flow objects
1653      */
1654     public void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) throws ElanException {
1655         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1656             return;
1657         }
1658         // dpId/externalNodeId will be available either in source or destination
1659         // based on the tunnel end point
1660         BigInteger dpId = null;
1661         NodeId externalNodeId = null;
1662         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1663             dpId = new BigInteger(externalTunnel.getSourceDevice());
1664             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1665         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1666             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1667             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1668         }
1669         if (dpId == null || externalNodeId == null) {
1670             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1671             return;
1672         }
1673
1674         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1675         if (dpnInterfaceLists == null) {
1676             return;
1677         }
1678         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1679         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1680             String elanName = elanDpns.getElanInstanceName();
1681             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1682
1683             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1684             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1685                     || dpnInterfaces.getInterfaces().isEmpty()) {
1686                 continue;
1687             }
1688             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1689
1690             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpId);
1691             // install L2gwDevices local macs in dpn.
1692             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1693             // Install dpn macs on external device
1694             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1695                     externalNodeId);
1696         }
1697         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1698     }
1699
1700     /**
1701      * Installs dpn macs in external device. first it checks if the physical
1702      * locator towards this dpn tep is present or not if the physical locator is
1703      * present go ahead and add the ucast macs otherwise update the mcast mac
1704      * entry to include this dpn tep ip and schedule the job to put ucast macs
1705      * once the physical locator is programmed in device
1706      *
1707      * @param elanName
1708      *            the elan name
1709      * @param lstElanInterfaceNames
1710      *            the lst Elan interface names
1711      * @param dpnId
1712      *            the dpn id
1713      * @param externalNodeId
1714      *            the external node id
1715      */
1716     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1717             NodeId externalNodeId) {
1718         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1719                 externalNodeId.getValue());
1720         if (elanL2GwDevice == null) {
1721             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1722             return;
1723         }
1724         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1725         if (dpnTepIp == null) {
1726             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1727                     externalNodeId);
1728             return;
1729         }
1730
1731         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1732         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1733                 LogicalDatastoreType.OPERATIONAL);
1734         boolean phyLocAlreadyExists =
1735                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1736                 dpnTepIp);
1737         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1738                 phyLocAlreadyExists, String.valueOf(dpnTepIp.getValue()), elanName, externalNodeId.getValue());
1739         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1740
1741         if (phyLocAlreadyExists) {
1742             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1743             return;
1744         }
1745         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1746         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1747     }
1748
1749     /**
1750      * Validate external tunnel state event.
1751      *
1752      * @param externalTunnel
1753      *            the external tunnel
1754      * @param intrf
1755      *            the intrf
1756      * @return true, if successful
1757      */
1758     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1759         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1760             String srcDevice = externalTunnel.getDestinationDevice();
1761             String destDevice = externalTunnel.getSourceDevice();
1762             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1763                     LogicalDatastoreType.CONFIGURATION);
1764             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1765                     otherEndPointExtTunnel);
1766             if (otherEndPointExtTunnel != null) {
1767                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1768                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1769                 if (otherEndPointInterfaceOperational) {
1770                     return true;
1771                 } else {
1772                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1773                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1774                 }
1775             }
1776         }
1777         return false;
1778     }
1779
1780     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1781         List<MatchInfo> mkMatches = new ArrayList<>();
1782         // Matching metadata
1783         mkMatches.add(
1784                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1785         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1786         return mkMatches;
1787     }
1788
1789     @Override
1790     protected ElanInterfaceManager getDataTreeChangeListener() {
1791         return this;
1792     }
1793
1794     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1795                                              BigInteger dpId) {
1796         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1797         elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId);
1798         try {
1799             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1800         } catch (InterruptedException e) {
1801             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1802         }
1803     }
1804 }