elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 142 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.slf4j.Logger;
 145 import org.slf4j.LoggerFactory;
 146
 147 /**
 148  * Class in charge of handling creations, modifications and removals of
 149  * ElanInterfaces.
 150  *
 151  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 152  */
 153 @Singleton
 154 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 155         implements RecoverableListener {
 156     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 157     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 158     private static final boolean SH_FLAG_SET = true;
 159     private static final boolean SH_FLAG_UNSET = false;
 160
 161     private final DataBroker broker;
 162     private final ManagedNewTransactionRunner txRunner;
 163     private final IMdsalApiManager mdsalManager;
 164     private final IInterfaceManager interfaceManager;
 165     private final IdManagerService idManager;
 166     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 167     private final INeutronVpnManager neutronVpnManager;
 168     private final ElanItmUtils elanItmUtils;
 169     private final ElanEtreeUtils elanEtreeUtils;
 170     private final ElanL2GatewayUtils elanL2GatewayUtils;
 171     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 172     private final ElanUtils elanUtils;
 173     private final JobCoordinator jobCoordinator;
 174     private final ElanInstanceCache elanInstanceCache;
 175     private final ElanInterfaceCache elanInterfaceCache;
 176     private final ElanGroupCache elanGroupCache;
 177
 178     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 179         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 180
 181     @Inject
 182     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 183                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 184                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 185                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 186                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 187                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 188                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 189                                 final ElanInstanceCache elanInstanceCache,
 190                                 final ElanInterfaceCache elanInterfaceCache,
 191                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 192                                 ElanGroupCache elanGroupCache,
 193                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 194         super(ElanInterface.class, ElanInterfaceManager.class);
 195         this.broker = dataBroker;
 196         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 197         this.idManager = managerService;
 198         this.mdsalManager = mdsalApiManager;
 199         this.interfaceManager = interfaceManager;
 200         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 201         this.neutronVpnManager = neutronVpnManager;
 202         this.elanItmUtils = elanItmUtils;
 203         this.elanEtreeUtils = elanEtreeUtils;
 204         this.elanL2GatewayUtils = elanL2GatewayUtils;
 205         this.elanUtils = elanUtils;
 206         this.jobCoordinator = jobCoordinator;
 207         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 208         this.elanInstanceCache = elanInstanceCache;
 209         this.elanInterfaceCache = elanInterfaceCache;
 210         this.elanGroupCache = elanGroupCache;
 211         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 212     }
 213
 214     @Override
 215     @PostConstruct
 216     public void init() {
 217         registerListener();
 218     }
 219
 220     @Override
 221     public void registerListener() {
 222         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 223     }
 224
 225     @Override
 226     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 227         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 228     }
 229
 230     @Override
 231     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 232         String interfaceName = del.getName();
 233         String elanInstanceName = del.getElanInstanceName();
 234         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 235         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 236             elanInterfaces.remove(del);
 237             if (elanInterfaces.isEmpty()) {
 238                 unProcessedElanInterfaces.remove(elanInstanceName);
 239             }
 240         }
 241         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 242         /*
 243          * Handling in case the elan instance is deleted.If the Elan instance is
 244          * deleted, there is no need to explicitly delete the elan interfaces
 245          */
 246         if (elanInfo == null) {
 247             return;
 248         }
 249         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 250         if (interfaceInfo == null && elanInfo.isExternal()) {
 251             // In deleting external network, the underlying ietf Inteface might have been removed
 252             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 253             // from Operational DS instead
 254             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 255         }
 256         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 257                 interfaceName, interfaceInfo, this);
 258         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 259     }
 260
 261     private static class RemoveElanInterfaceHolder {
 262         boolean isLastElanInterface = false;
 263         BigInteger dpId = null;
 264     }
 265
 266     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 267     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 268             InterfaceInfo interfaceInfo) {
 269         String elanName = elanInfo.getElanInstanceName();
 270         long elanTag = elanInfo.getElanTag();
 271         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 272         List<ListenableFuture<Void>> futures = new ArrayList<>();
 273         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 274         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 275             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 276             if (elanState == null) {
 277                 return;
 278             }
 279             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 280                 List<String> elanInterfaces = elanState.getElanInterfaces();
 281                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 282                     holder.isLastElanInterface = true;
 283                 }
 284                 if (interfaceInfo != null) {
 285                     holder.dpId = interfaceInfo.getDpId();
 286                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 287                         interfaceName, elanTag, interfaceTx);
 288                     /*
 289                      * If there are not elan ports, remove the unknown dmac, terminating
 290                      * service table flows, remote/local bc group
 291                      */
 292                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 293                         || dpnInterfaces.getInterfaces().isEmpty()) {
 294                         // No more Elan Interfaces in this DPN
 295                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 296                             holder.dpId);
 297                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 299                         }
 300                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 301                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 302                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 303                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 304                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 305                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 306                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 307                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 308                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 309                             }
 310                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 311                         }
 312                     } else {
 313                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 314                     }
 315                 }
 316             }));
 317         }));
 318         futures.forEach(ElanUtils::waitForTransactionToComplete);
 319
 320         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 321                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 322         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 323                 ElanConstants.JOB_MAX_RETRIES);
 324
 325         return futures;
 326     }
 327
 328     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 329             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 330             throws ExecutionException, InterruptedException {
 331         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 332         if (etreeLeafTag != null) {
 333             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 334             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 335         }
 336     }
 337
 338     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 339             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 340             throws ExecutionException, InterruptedException {
 341         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 342         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 343     }
 344
 345     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 346             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 347             throws ExecutionException, InterruptedException {
 348         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 349         if (etreeInstance != null) {
 350             BigInteger dpnId = interfaceInfo.getDpId();
 351             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 352             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 353             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 354         }
 355     }
 356
 357     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 358             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 359             throws ExecutionException, InterruptedException {
 360         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 361         if (etreeInstance != null) {
 362             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 363             BigInteger dpnId = interfaceInfo.getDpId();
 364             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 365             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 366             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 367         }
 368     }
 369
 370     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 371             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 372         String elanName = elanInfo.getElanInstanceName();
 373         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 374         if (elanState == null) {
 375             return elanState;
 376         }
 377         List<String> elanInterfaces = elanState.getElanInterfaces();
 378         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 379         if (!isRemoved) {
 380             return elanState;
 381         }
 382
 383         if (elanInterfaces.isEmpty()) {
 384             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 385             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 386             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 387             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 388         } else {
 389             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 390                     .withKey(new ElanKey(elanName)).build();
 391             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 392         }
 393         return elanState;
 394     }
 395
 396     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 397             throws ReadFailedException {
 398         // removing the ElanInterface from the config data_store if interface is
 399         // not present in Interface config DS
 400         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 401             interfaceName) == null
 402                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 403             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 404         }
 405     }
 406
 407     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 408             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 409         String elanName = elanInfo.getElanInstanceName();
 410         List<ListenableFuture<Void>> futures = new ArrayList<>();
 411         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 412             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 413                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 414                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 415                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 416                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 417                 if (interfaceInfo != null) {
 418                     if (existingElanInterfaceMac.isPresent()) {
 419                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 420                         if (existingMacEntries != null) {
 421                             List<PhysAddress> macAddresses = new ArrayList<>();
 422                             for (MacEntry macEntry : existingMacEntries) {
 423                                 PhysAddress macAddress = macEntry.getMacAddress();
 424                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 425                                         macAddress.getValue(), interfaceName);
 426                                 Optional<MacEntry> macEntryOptional =
 427                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 428                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 429                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 430                                 }
 431                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 432                                 macAddresses.add(macAddress);
 433                             }
 434
 435                             // Removing all those MACs from External Devices belonging
 436                             // to this ELAN
 437                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 438                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 439                             }
 440                         }
 441                     }
 442                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 443                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 444                 } else if (existingElanInterfaceMac.isPresent()) {
 445                     // Interface does not exist in ConfigDS, so lets remove everything
 446                     // about that interface related to Elan
 447                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 448                     if (macEntries != null) {
 449                         for (MacEntry macEntry : macEntries) {
 450                             PhysAddress macAddress = macEntry.getMacAddress();
 451                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 452                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 453                             }
 454                         }
 455                     }
 456                 }
 457                 if (existingElanInterfaceMac.isPresent()) {
 458                     interfaceTx.delete(elanInterfaceId);
 459                 }
 460                 unbindService(interfaceName, flowTx);
 461                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 462             }));
 463         }));
 464         return futures;
 465     }
 466
 467     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 468                                                                          String interfaceName, long elanTag,
 469                                                                          TypedReadWriteTransaction<Operational> tx)
 470             throws ExecutionException, InterruptedException {
 471         // FIXME: pass in and use ElanInstanceKey instead?
 472         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 473         lock.lock();
 474         try {
 475             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 476             if (dpnInterfaces != null) {
 477                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 478                 if (interfaceLists != null) {
 479                     interfaceLists.remove(interfaceName);
 480                 }
 481
 482                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 483                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 484                     deleteElanDpnInterface(elanName, dpId, tx);
 485                 } else {
 486                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 487                 }
 488             }
 489             return dpnInterfaces;
 490         } finally {
 491             lock.unlock();
 492         }
 493     }
 494
 495     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 496         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 497         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 498             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 499                 BigInteger currentDpId = dpnInterface.getDpId();
 500                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 501                     for (String elanInterface : dpnInterface.getInterfaces()) {
 502                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 503                         if (macs == null || macs.getMacEntry() == null) {
 504                             continue;
 505                         }
 506                         for (MacEntry mac : macs.getMacEntry()) {
 507                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 508                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 509                         }
 510                     }
 511                 }
 512             }
 513         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 514     }
 515
 516     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 517             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 518         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 519         if (etreeLeafTag != null) {
 520             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 521         }
 522     }
 523
 524     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 525             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 526         mdsalManager
 527                 .removeFlow(confTx, dpId,
 528                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 529                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 530                                         mac.getMacAddress().getValue(), elanTag)));
 531     }
 532
 533     /*
 534     * Possible Scenarios for update
 535     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 536         then all {1,2,3,4} should be removed
 537
 538         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 539         then all {1,2,3,4} should be added
 540
 541         c. if orig = {1,2,3,4} updated={2,3,4}
 542         then 1 should be removed
 543
 544         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 545         then we should just add 5
 546
 547         e. if orig = {1,2,3,4} updated={2,3,4,5}
 548         then 1 should be removed , 5 should be added
 549     * */
 550     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 551     @Override
 552     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 553         // updating the static-Mac Entries for the existing elanInterface
 554         String elanName = update.getElanInstanceName();
 555         String interfaceName = update.getName();
 556         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 557
 558         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 559         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 560         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 561         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 562
 563         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 564                 deletedEntry.getMacAddress()));
 565
 566         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 567         * Hence add the macentries for the same.*/
 568         for (StaticMacEntries staticMacEntry : updatedEntries) {
 569             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 570                     staticMacEntry.getMacAddress());
 571             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 572                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 573                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 574                     if (existingMacEntry.isPresent()) {
 575                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 576                                 existingMacEntry.get(), elanName);
 577                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 578                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 579                             tx);
 580                     } else {
 581                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 582                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 583                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 584                             elanName, interfaceName, staticMacEntry, tx);
 585                     }
 586                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 587         }
 588     }
 589
 590     @Override
 591     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 592         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 593         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 594             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 595             String interfaceName = elanInterfaceAdded.getName();
 596             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 597             if (interfaceInfo == null) {
 598                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 599                 return;
 600             }
 601             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 602
 603             if (elanInstance == null) {
 604                 // Add the ElanInstance in the Configuration data-store
 605                 List<String> elanInterfaces = new ArrayList<>();
 606                 elanInterfaces.add(interfaceName);
 607                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 608                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 609                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 610                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 611             }
 612
 613             Long elanTag = elanInstance.getElanTag();
 614             // If elan tag is not updated, then put the elan interface into
 615             // unprocessed entry map and entry. Let entries
 616             // in this map get processed during ELAN update DCN.
 617             if (elanTag == null || elanTag == 0L) {
 618                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 619                 if (elanInterfaces == null) {
 620                     elanInterfaces = new ConcurrentLinkedQueue<>();
 621                 }
 622                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 623                     elanInterfaces.add(elanInterfaceAdded);
 624                 }
 625                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 626                 return;
 627             }
 628             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 629                     interfaceInfo, elanInstance, this);
 630             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 631         }), LOG, "Error processing added ELAN interface");
 632     }
 633
 634     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 635         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 636         List<ListenableFuture<Void>> futures = new ArrayList<>();
 637         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 638         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 639             return futures;
 640         }
 641         for (ElanInterface elanInterface : elanInterfaces) {
 642             String interfaceName = elanInterface.getName();
 643             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 644             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 645         }
 646         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 647         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 648                 elanInstance.getElanInstanceName());
 649         return futures;
 650     }
 651
 652     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 653             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 654         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 655                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 656         List<DpnInterfaces> dpnInterfaceLists = null;
 657         if (elanDpnInterfacesList != null) {
 658             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 659         }
 660         if (dpnInterfaceLists == null) {
 661             dpnInterfaceLists = new ArrayList<>();
 662         }
 663         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 664             BigInteger dstDpId = interfaceInfo.getDpId();
 665             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 666                 continue;
 667             }
 668             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 669             for (String remoteIf : remoteElanInterfaces) {
 670                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 671                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 672                 if (elanIfMac == null || remoteInterface == null) {
 673                     continue;
 674                 }
 675                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 676                 if (remoteMacEntries != null) {
 677                     for (MacEntry macEntry : remoteMacEntries) {
 678                         String macAddress = macEntry.getMacAddress().getValue();
 679                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 680                                 dstDpId, elanInstance.getElanInstanceName());
 681                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 682                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 683                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 684                     }
 685                 }
 686             }
 687         }
 688     }
 689
 690     private static class AddElanInterfaceHolder {
 691         private DpnInterfaces dpnInterfaces = null;
 692         private boolean isFirstInterfaceInDpn = false;
 693         private BigInteger dpId;
 694     }
 695
 696     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 697     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 698             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 699         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 700         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 701         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 702
 703         String interfaceName = elanInterface.getName();
 704         String elanInstanceName = elanInterface.getElanInstanceName();
 705         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 706
 707         List<ListenableFuture<Void>> futures = new ArrayList<>();
 708         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 709         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 710             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 711             if (elanInfo == null) {
 712                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 713                 List<String> elanInterfaces = new ArrayList<>();
 714                 elanInterfaces.add(interfaceName);
 715                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 716                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 717                         operTx)));
 718             } else {
 719                 createElanStateList(elanInstanceName, interfaceName, operTx);
 720             }
 721             // Specific actions to the DPN where the ElanInterface has been added,
 722             // for example, programming the
 723             // External tunnel table if needed or adding the ElanInterface to the
 724             // DpnInterfaces in the operational DS.
 725             holder.dpId = interfaceInfo.getDpId();
 726             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 727                 // FIXME: use elanInstaince.key() instead?
 728                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 729                 lock.lock();
 730                 try {
 731                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 732                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 733                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 734                     if (ElanUtils.isVlan(elanInstance)) {
 735                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 736                             elanInstanceName, existingElanDpnInterfaces);
 737                     } else {
 738                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 739                     }
 740                     if (holder.isFirstInterfaceInDpn) {
 741                         // ELAN's 1st ElanInterface added to this DPN
 742                         if (!existingElanDpnInterfaces.isPresent()) {
 743                             holder.dpnInterfaces =
 744                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 745                         } else {
 746                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 747                             List<String> elanInterfaces =
 748                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 749                             elanInterfaces.add(interfaceName);
 750                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 751                                 elanInterfaces, operTx);
 752                         }
 753                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 754                                 interfaceName, elanInstanceName, holder.dpId);
 755                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 756                         // table, but only if Elan has VNI
 757                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 758                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 759                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 760                         }
 761                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 762                             interfaceName);
 763                     } else {
 764                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 765                         List<String> elanInterfaces =
 766                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 767                         elanInterfaces.add(interfaceName);
 768                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 769                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 770                                 interfaceName);
 771                         }
 772                         holder.dpnInterfaces =
 773                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 774                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 775                                 interfaceName, elanInstanceName, holder.dpId);
 776                     }
 777                 } finally {
 778                     lock.unlock();
 779                 }
 780             }
 781
 782             // add code to install Local/Remote BC group, unknow DMAC entry,
 783             // terminating service table flow entry
 784             // call bindservice of interfacemanager to create ingress table flow
 785             // enty.
 786             // Add interface to the ElanInterfaceForwardingEntires Container
 787             createElanInterfaceTablesList(interfaceName, operTx);
 788         }));
 789         futures.forEach(ElanUtils::waitForTransactionToComplete);
 790         futures.add(
 791             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 792                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 793                     holder.isFirstInterfaceInDpn, confTx))));
 794
 795         // add the vlan provider interface to remote BC group for the elan
 796         // for internal vlan networks
 797         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 798             if (interfaceManager.isExternalInterface(interfaceName)) {
 799                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 800                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 801             }
 802         }
 803         if (holder.isFirstInterfaceInDpn) {
 804             // ELAN's 1st ElanInterface added to this DPN
 805             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 806             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 807                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 808                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 809             }));
 810         } else {
 811             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 812         }
 813
 814         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 815                 holder.isFirstInterfaceInDpn, elanInterface);
 816         return futures;
 817     }
 818
 819     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 820     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 821             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 822         String elanInstanceName = elanInstance.getElanInstanceName();
 823         String interfaceName = elanInterface.getName();
 824         List<ListenableFuture<Void>> futures = new ArrayList<>();
 825         BigInteger dpId = interfaceInfo.getDpId();
 826         boolean isInterfaceOperational = isOperational(interfaceInfo);
 827         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 828             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 829                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 830                     isFirstInterfaceInDpn, confTx);
 831
 832                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 833                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 834
 835                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 836                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 837                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 838                             staticMacEntry.getMacAddress());
 839                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 840                             LogicalDatastoreType.OPERATIONAL, macId);
 841                         if (existingMacEntry.isPresent()) {
 842                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 843                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 844                                 existingMacEntry.get(), operTx);
 845                         } else {
 846                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 847                                 interfaceName, staticMacEntry, operTx);
 848                         }
 849
 850                         if (isInterfaceOperational) {
 851                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 852                             // DPNs
 853                             String macAddress = staticMacEntry.getMacAddress().getValue();
 854                             LOG.info(
 855                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 856                                     + " {}",
 857                                 macAddress, elanInstanceName, interfaceName);
 858                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 859                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 860                         }
 861                     }
 862
 863                     if (isInterfaceOperational) {
 864                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 865                         // on purpose.
 866                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 867                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 868                         }
 869                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 870                             staticMacAddresses);
 871                     }
 872                 }
 873             }));
 874         }));
 875         futures.forEach(ElanUtils::waitForTransactionToComplete);
 876         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 877             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 878             try {
 879                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 880                 if (neutronPort != null) {
 881                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 882                 }
 883             } catch (IllegalArgumentException ex) {
 884                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 885             }
 886         }
 887         return futures;
 888     }
 889
 890     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 891             PhysAddress physAddress) {
 892         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 893         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 894         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 895                 LogicalDatastoreType.OPERATIONAL, macId);
 896
 897         if (!existingMacEntry.isPresent()) {
 898             return;
 899         }
 900
 901         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 902                 .withKey(new MacEntryKey(physAddress)).build();
 903         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 904                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 905     }
 906
 907     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 908             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 909         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 910         if (!existingElanDpnInterfaces.isPresent()) {
 911             return true;
 912         }
 913         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 914             return false;
 915         }
 916         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 917         int dummyInterfaceCount =  0;
 918         List<String> interfaces = dpnInterfaces.getInterfaces();
 919         if (interfaces == null) {
 920             return true;
 921         }
 922         if (interfaces.contains(routerPortUuid)) {
 923             dummyInterfaceCount++;
 924         }
 925         if (interfaces.contains(elanInstanceName)) {
 926             dummyInterfaceCount++;
 927         }
 928         return interfaces.size() == dummyInterfaceCount;
 929     }
 930
 931     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 932             PhysAddress physAddress) {
 933         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 934                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 935     }
 936
 937     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 938             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 939         if (!isOperational(interfaceInfo)) {
 940             LOG.warn("Interface {} is not operational", elanInterface.getName());
 941             return;
 942         }
 943         BigInteger dpId = interfaceInfo.getDpId();
 944         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 945             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 946         }
 947         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 948         if (isFirstInterfaceInDpn) {
 949             // Terminating Service , UnknownDMAC Table.
 950             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 951             // for ELAN Instance is VxLAN
 952             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 953                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 954             }
 955             setupUnknownDMacTable(elanInstance, dpId, confTx);
 956             /*
 957              * Install remote DMAC flow. This is required since this DPN is
 958              * added later to the elan instance and remote DMACs of other
 959              * interfaces in this elan instance are not present in the current
 960              * dpn.
 961              */
 962             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 963                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 964                         elanInstance.getElanInstanceName());
 965                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 966             }
 967         }
 968         // bind the Elan service to the Interface
 969         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 970     }
 971
 972     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 973             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 974         if (!isOperational(interfaceInfo)) {
 975             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
 976                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
 977             return;
 978         }
 979         // LocalBroadcast Group creation with elan-Interfaces
 980         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
 981                 dpnInterfaces.getDpId());
 982         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 983         if (isFirstInterfaceInDpn) {
 984             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 985             try {
 986                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 987             } catch (InterruptedException e1) {
 988                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 989             }
 990         }
 991     }
 992
 993     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, BigInteger dpnId) {
 994         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
 995         return InstanceIdentifier.builder(Nodes.class)
 996                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
 997                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
 998                 .augmentation(FlowCapableNode.class)
 999                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1000     }
1001
1002     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1003                                                               InterfaceInfo interfaceInfo,
1004                                                               DpnInterfaces dpnInterfaces,
1005                                                               boolean isFirstInterfaceInDpn,
1006                                                               ElanInterface elanInterface) {
1007         if (!isOperational(interfaceInfo)) {
1008             LOG.debug("Interface {} is not operational", elanInterface.getName());
1009             return;
1010         }
1011         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1012         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1013                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1014         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1015         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1016             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1017         });
1018     }
1019
1020     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1021             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1022         int ifTag = interfaceInfo.getInterfaceTag();
1023         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1024                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
1025                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1026                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1027                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1028
1029         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1030         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1031                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1032
1033         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1034                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1035                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1036                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1037
1038         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1039         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1040                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1041     }
1042
1043     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1044             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1045         int ifTag = interfaceInfo.getInterfaceTag();
1046         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1047             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1048
1049         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1050
1051         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1052             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1053             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1054             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1055
1056         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1057     }
1058
1059     private static List<MatchInfo> buildMatchesForVni(Long vni) {
1060         List<MatchInfo> mkMatches = new ArrayList<>();
1061         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1062         mkMatches.add(match);
1063         return mkMatches;
1064     }
1065
1066     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1067         List<InstructionInfo> mkInstructions = new ArrayList<>();
1068         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1069         return mkInstructions;
1070     }
1071
1072     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1073         List<MatchInfo> mkMatches = new ArrayList<>();
1074         // Matching metadata
1075         mkMatches.add(new MatchMetadata(
1076                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1077         return mkMatches;
1078     }
1079
1080     /**
1081      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1082      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1083      *
1084      * @param elanTag
1085      *            elanTag to be written in metadata when flow is selected
1086      * @return the instructions ready to be installed in a flow
1087      */
1088     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1089         List<InstructionInfo> mkInstructions = new ArrayList<>();
1090         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1091                 .getElanMetadataMask()));
1092         /* applicable for EXTERNAL_TUNNEL_TABLE only
1093         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1094         */
1095         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1096         return mkInstructions;
1097     }
1098
1099     // Install DMAC entry on dst DPN
1100     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1101             BigInteger dstDpId) {
1102         String interfaceName = interfaceInfo.getInterfaceName();
1103         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1104         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1105             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1106             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1107                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1108                     for (MacEntry macEntry : macEntries) {
1109                         String macAddress = macEntry.getMacAddress().getValue();
1110                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1111                             interfaceName);
1112                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag(), macAddress,
1113                             interfaceInfo.getDpId())) {
1114                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1115                                 macAddress);
1116                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1117                         }
1118                     }
1119                 })));
1120         }
1121         return emptyList();
1122     }
1123
1124     private static void createDropBucket(List<Bucket> listBucket) {
1125         List<Action> actionsInfos = new ArrayList<>();
1126         actionsInfos.add(new ActionDrop().buildAction());
1127         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1128                 MDSALUtil.WATCH_GROUP);
1129         listBucket.add(dropBucket);
1130     }
1131
1132     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1133             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1134         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1135         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1136     }
1137
1138     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1139             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1140         List<Bucket> listBucket = new ArrayList<>();
1141         int bucketId = 0;
1142         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1143
1144         List<String> interfaces = new ArrayList<>();
1145         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1146             interfaces = newDpnInterface.getInterfaces();
1147         }
1148         for (String ifName : interfaces) {
1149             // In case if there is a InterfacePort in the cache which is not in
1150             // operational state, skip processing it
1151             InterfaceInfo ifInfo = interfaceManager
1152                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1153             if (!isOperational(ifInfo)) {
1154                 continue;
1155             }
1156
1157             if (!interfaceManager.isExternalInterface(ifName)) {
1158                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1159                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1160                 bucketId++;
1161             }
1162         }
1163
1164         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1165                 MDSALUtil.buildBucketLists(listBucket));
1166         LOG.trace("installing the localBroadCast Group:{}", group);
1167         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1168     }
1169
1170     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1171             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1172         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1173         if (etreeInstance != null) {
1174             List<Bucket> listBucket = new ArrayList<>();
1175             int bucketId = 0;
1176
1177             List<String> interfaces = new ArrayList<>();
1178             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1179                 interfaces = newDpnInterface.getInterfaces();
1180             }
1181             for (String ifName : interfaces) {
1182                 // In case if there is a InterfacePort in the cache which is not
1183                 // in
1184                 // operational state, skip processing it
1185                 InterfaceInfo ifInfo = interfaceManager
1186                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1187                 if (!isOperational(ifInfo)) {
1188                     continue;
1189                 }
1190
1191                 if (!interfaceManager.isExternalInterface(ifName)) {
1192                     // only add root interfaces
1193                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1194                 }
1195             }
1196
1197             if (listBucket.isEmpty()) { // No Buckets
1198                 createDropBucket(listBucket);
1199             }
1200
1201             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1202             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1203             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1204                     MDSALUtil.buildBucketLists(listBucket));
1205             LOG.trace("installing the localBroadCast Group:{}", group);
1206             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1207         }
1208     }
1209
1210     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1211             InterfaceInfo ifInfo) {
1212         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1213         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1214             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1215                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1216             bucketId++;
1217         }
1218         return bucketId;
1219     }
1220
1221     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1222             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1223             throws ExecutionException, InterruptedException {
1224         BigInteger dpnId = interfaceInfo.getDpId();
1225         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1226         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1227         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1228     }
1229
1230     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1231             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1232             throws ExecutionException, InterruptedException {
1233         BigInteger dpnId = interfaceInfo.getDpId();
1234         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1235         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1236         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1237     }
1238
1239     /**
1240      * Installs a flow in the External Tunnel table consisting in translating
1241      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1242      * elanTag that will be used later in the ELANs pipeline.
1243      * @param dpnId the dpn id
1244      */
1245     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1246             TypedWriteTransaction<Configuration> confTx) {
1247         long elanTag = elanInfo.getElanTag();
1248         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1249                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1250                 elanInfo.getElanInstanceName(), // flowName
1251                 0, // idleTimeout
1252                 0, // hardTimeout
1253                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1254                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1255                 getInstructionsIntOrExtTunnelTable(elanTag));
1256
1257         mdsalManager.addFlow(confTx, flowEntity);
1258     }
1259
1260     /**
1261      * Removes, from External Tunnel table, the flow that translates from VNI to
1262      * elanTag. Important: ensure this method is only called whenever there is
1263      * no other ElanInterface in the specified DPN
1264      *
1265      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1266      */
1267     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1268             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1269         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1270         // last ElanInstance plus
1271         // adding a new one does (almost at the same time) are executed in that
1272         // exact order
1273
1274         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1275         FlowEntity flowEntity = new FlowEntityBuilder()
1276             .setDpnId(dpnId)
1277             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1278             .setFlowId(flowId)
1279             .build();
1280         mdsalManager.removeFlow(confTx, flowEntity);
1281     }
1282
1283     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1284             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1285         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1286         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1287     }
1288
1289     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1290             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1291         List<? extends MatchInfoBase> listMatchInfoBase;
1292         List<InstructionInfo> instructionInfos;
1293         long serviceId;
1294         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1295             serviceId = elanTag;
1296             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1297             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1298         } else {
1299             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1300             listMatchInfoBase = buildMatchesForVni(serviceId);
1301             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1302         }
1303         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1304                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1305                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1306                 instructionInfos);
1307         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1308         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1309                 elanInfo.getElanInstanceName(), elanTag);
1310     }
1311
1312     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1313             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1314         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1315         if (etreeInstance != null) {
1316             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1317         }
1318     }
1319
1320     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1321             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1322         long elanTag = elanInfo.getElanTag();
1323         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1324         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1325         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1326     }
1327
1328     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1329             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1330         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1331         if (etreeLeafTag != null) {
1332             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1333             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1334             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1335         }
1336     }
1337
1338     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1339             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1340         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1341                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1342                 5, elanInfo.getElanInstanceName(), 0, 0,
1343                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1344                 getMatchesForElanTag(elanTag, /* SH flag */false),
1345                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1346
1347         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1348         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1349                 dpId, elanInfo.getElanInstanceName(), elanTag);
1350     }
1351
1352     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1353             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1354         // only if ELAN can connect to external network, perform the following
1355
1356         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1357             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1358                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1359                     5, elanInfo.getElanInstanceName(), 0, 0,
1360                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1361                     getMatchesForElanTag(elanTag, /* SH flag */true),
1362                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1363             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1364             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1365                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1366         }
1367     }
1368
1369
1370     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1371             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1372             throws ExecutionException, InterruptedException {
1373         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1374                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1375         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1376
1377         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1378             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1379                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1380                     .build();
1381             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1382         }
1383     }
1384
1385     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1386         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1387     }
1388
1389     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1390             TypedWriteTransaction<Configuration> tx) {
1391         if (isStandardElanService(elanInterface)) {
1392             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1393                     elanInterface.getName(), lportTag, tx);
1394         } else { // Etree service
1395             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1396         }
1397     }
1398
1399     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1400             TypedWriteTransaction<Configuration> tx) {
1401         int instructionKey = 0;
1402         List<Instruction> instructions = new ArrayList<>();
1403         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1404                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1405
1406         List<Action> actions = new ArrayList<>();
1407         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1408                 lportTag).buildAction());
1409         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1410                 elanTag).buildAction());
1411         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1412
1413         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1414                 ++instructionKey));
1415
1416         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1417         BoundServices serviceInfo = ElanUtils.getBoundServices(
1418                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1419                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1420         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1421         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1422                 bindServiceId);
1423         if (!existingElanService.isPresent()) {
1424             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1425             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1426         }
1427     }
1428
1429     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1430             TypedWriteTransaction<Configuration> tx) {
1431         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1432             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1433                     lportTag, tx);
1434         } else {
1435             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1436             if (etreeInstance == null) {
1437                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1438                         elanInterface.getName(), elanInfo.getElanInstanceName());
1439             } else {
1440                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1441                         elanInterface.getName(), lportTag, tx);
1442             }
1443         }
1444     }
1445
1446     private static boolean isStandardElanService(ElanInterface elanInterface) {
1447         return elanInterface.augmentation(EtreeInterface.class) == null;
1448     }
1449
1450     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1451             throws ExecutionException, InterruptedException {
1452         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1453         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1454         if (tx.read(bindServiceId).get().isPresent()) {
1455             tx.delete(bindServiceId);
1456         }
1457     }
1458
1459     private static String getFlowRef(long tableId, long elanTag) {
1460         return String.valueOf(tableId) + elanTag;
1461     }
1462
1463     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1464         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1465                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1466     }
1467
1468     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1469         return String.valueOf(tableId) + elanTag + shFlag;
1470     }
1471
1472     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1473         List<Action> listAction = new ArrayList<>();
1474         int actionKey = 0;
1475         listAction.add(
1476             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1477         actionKey++;
1478         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1479         return listAction;
1480     }
1481
1482     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1483             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1484         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1485                 .withKey(new DpnInterfacesKey(dpId)).build();
1486         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1487                 CREATE_MISSING_PARENTS);
1488         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1489                 interfaceNames);
1490         return dpnInterface;
1491     }
1492
1493     /**
1494      * Delete elan dpn interface from operational DS.
1495      *
1496      * @param elanInstanceName
1497      *            the elan instance name
1498      * @param dpId
1499      *            the dp id
1500      */
1501     private static void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1502             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1503         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1504                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1505         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1506         if (dpnInterfaces.isPresent()) {
1507             tx.delete(dpnInterfacesId);
1508         }
1509     }
1510
1511     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1512             BigInteger dpId, TypedWriteTransaction<Operational> tx) {
1513         List<String> interfaceNames = new ArrayList<>();
1514         interfaceNames.add(interfaceName);
1515         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1516                 .withKey(new DpnInterfacesKey(dpId)).build();
1517         return dpnInterface;
1518     }
1519
1520     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1521             throws ExecutionException, InterruptedException {
1522         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1523                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1524         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1525         // Adding new Elan Interface Port to the operational DataStore without
1526         // Static-Mac Entries..
1527         if (!interfaceMacTables.isPresent()) {
1528             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1529                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1530             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1531                     CREATE_MISSING_PARENTS);
1532             LOG.trace("Created interface MAC table for interface: {}", interfaceName);
1533         }
1534     }
1535
1536     private static void createElanStateList(String elanInstanceName, String interfaceName,
1537             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1538         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1539         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1540         // Adding new Elan Interface Port to the operational DataStore without
1541         // Static-Mac Entries..
1542         if (elanInterfaceLists.isPresent()) {
1543             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1544             if (interfaceLists == null) {
1545                 interfaceLists = new ArrayList<>();
1546             }
1547             interfaceLists.add(interfaceName);
1548             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1549                     .withKey(new ElanKey(elanInstanceName)).build();
1550             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1551             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1552                     interfaceLists);
1553         }
1554     }
1555
1556     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1557         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1558     }
1559
1560     @SuppressWarnings("checkstyle:IllegalCatch")
1561     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1562         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1563         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1564                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1565         if (dpnInterfaceLists == null) {
1566             return;
1567         }
1568         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1569         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1570             int cnt = 0;
1571             String elanName = elanDpns.getElanInstanceName();
1572             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1573             if (elanInfo == null) {
1574                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1575                         + "skipping this ELAN for tunnel handling", elanName);
1576                 continue;
1577             }
1578             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1579                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1580                 continue;
1581             }
1582             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1583             if (dpnInterfaces == null) {
1584                 continue;
1585             }
1586             DpnInterfaces dstDpnIf = null;
1587             for (DpnInterfaces dpnIf : dpnInterfaces) {
1588                 BigInteger dpnIfDpId = dpnIf.getDpId();
1589                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1590                     cnt++;
1591                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1592                     cnt++;
1593                     dstDpnIf = dpnIf;
1594                 }
1595             }
1596             if (cnt == 2) {
1597                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1598                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1599                 jobCoordinator.enqueueJob(elanName, () -> {
1600                     // update Remote BC Group
1601                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1602                     try {
1603                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1604                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1605                                 confTx)).get();
1606                     } catch (RuntimeException e) {
1607                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1608                     }
1609                     Set<String> interfaceLists = new HashSet<>();
1610                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1611                     for (String ifName : interfaceLists) {
1612                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1613                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1614                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1615                             if (isOperational(interfaceInfo)) {
1616                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1617                             }
1618                             return emptyList();
1619                         }, ElanConstants.JOB_MAX_RETRIES);
1620                     }
1621                     return emptyList();
1622                 }, ElanConstants.JOB_MAX_RETRIES);
1623             }
1624
1625         }
1626     }
1627
1628     /**
1629      * Handle external tunnel state event.
1630      *
1631      * @param externalTunnel
1632      *            the external tunnel
1633      * @param intrf
1634      *            the interface
1635      */
1636     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1637         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1638             return;
1639         }
1640         // dpId/externalNodeId will be available either in source or destination
1641         // based on the tunnel end point
1642         BigInteger dpId = null;
1643         NodeId externalNodeId = null;
1644         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1645             dpId = new BigInteger(externalTunnel.getSourceDevice());
1646             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1647         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1648             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1649             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1650         }
1651         if (dpId == null || externalNodeId == null) {
1652             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1653             return;
1654         }
1655
1656         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1657         if (dpnInterfaceLists == null) {
1658             return;
1659         }
1660         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1661         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1662             String elanName = elanDpns.getElanInstanceName();
1663             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1664
1665             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1666             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1667                     || dpnInterfaces.getInterfaces().isEmpty()) {
1668                 continue;
1669             }
1670             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1671
1672             final BigInteger finalDpId = dpId;
1673             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1674                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1675                 "Error setting up ELAN BGs");
1676             // install L2gwDevices local macs in dpn.
1677             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1678             // Install dpn macs on external device
1679             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1680                     externalNodeId);
1681         }
1682         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1683     }
1684
1685     /**
1686      * Installs dpn macs in external device. first it checks if the physical
1687      * locator towards this dpn tep is present or not if the physical locator is
1688      * present go ahead and add the ucast macs otherwise update the mcast mac
1689      * entry to include this dpn tep ip and schedule the job to put ucast macs
1690      * once the physical locator is programmed in device
1691      *
1692      * @param elanName
1693      *            the elan name
1694      * @param lstElanInterfaceNames
1695      *            the lst Elan interface names
1696      * @param dpnId
1697      *            the dpn id
1698      * @param externalNodeId
1699      *            the external node id
1700      */
1701     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1702             NodeId externalNodeId) {
1703         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1704                 externalNodeId.getValue());
1705         if (elanL2GwDevice == null) {
1706             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1707             return;
1708         }
1709         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1710         if (dpnTepIp == null) {
1711             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1712                     externalNodeId);
1713             return;
1714         }
1715
1716         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1717         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1718                 LogicalDatastoreType.OPERATIONAL);
1719         boolean phyLocAlreadyExists =
1720                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1721                 dpnTepIp);
1722         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1723                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1724         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1725
1726         if (phyLocAlreadyExists) {
1727             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1728             return;
1729         }
1730         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1731         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1732     }
1733
1734     /**
1735      * Validate external tunnel state event.
1736      *
1737      * @param externalTunnel
1738      *            the external tunnel
1739      * @param intrf
1740      *            the intrf
1741      * @return true, if successful
1742      */
1743     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1744         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1745             String srcDevice = externalTunnel.getDestinationDevice();
1746             String destDevice = externalTunnel.getSourceDevice();
1747             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1748                     LogicalDatastoreType.CONFIGURATION);
1749             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1750                     otherEndPointExtTunnel);
1751             if (otherEndPointExtTunnel != null) {
1752                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1753                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1754                 if (otherEndPointInterfaceOperational) {
1755                     return true;
1756                 } else {
1757                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1758                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1759                 }
1760             }
1761         }
1762         return false;
1763     }
1764
1765     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1766         List<MatchInfo> mkMatches = new ArrayList<>();
1767         // Matching metadata
1768         mkMatches.add(
1769                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1770         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1771         return mkMatches;
1772     }
1773
1774     @Override
1775     protected ElanInterfaceManager getDataTreeChangeListener() {
1776         return this;
1777     }
1778
1779     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1780                                              BigInteger dpId) {
1781         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1782         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1783             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1784                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1785         try {
1786             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1787         } catch (InterruptedException e) {
1788             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1789         }
1790     }
1791 }