elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 135 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 /**
 141  * Class in charge of handling creations, modifications and removals of
 142  * ElanInterfaces.
 143  *
 144  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 145  */
 146 @Singleton
 147 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 148         implements RecoverableListener {
 149     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final boolean SH_FLAG_SET = true;
 152     private static final boolean SH_FLAG_UNSET = false;
 153
 154     private final DataBroker broker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalManager;
 157     private final IInterfaceManager interfaceManager;
 158     private final IdManagerService idManager;
 159     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 160     private final INeutronVpnManager neutronVpnManager;
 161     private final ElanItmUtils elanItmUtils;
 162     private final ElanEtreeUtils elanEtreeUtils;
 163     private final ElanL2GatewayUtils elanL2GatewayUtils;
 164     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 165     private final ElanUtils elanUtils;
 166     private final JobCoordinator jobCoordinator;
 167     private final ElanInstanceCache elanInstanceCache;
 168     private final ElanInterfaceCache elanInterfaceCache;
 169
 170     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 171         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 172
 173     @Inject
 174     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 175                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 176                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 177                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 178                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 179                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 180                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 181                                 final ElanInstanceCache elanInstanceCache,
 182                                 final ElanInterfaceCache elanInterfaceCache,
 183                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 184                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 185         super(ElanInterface.class, ElanInterfaceManager.class);
 186         this.broker = dataBroker;
 187         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 188         this.idManager = managerService;
 189         this.mdsalManager = mdsalApiManager;
 190         this.interfaceManager = interfaceManager;
 191         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 192         this.neutronVpnManager = neutronVpnManager;
 193         this.elanItmUtils = elanItmUtils;
 194         this.elanEtreeUtils = elanEtreeUtils;
 195         this.elanL2GatewayUtils = elanL2GatewayUtils;
 196         this.elanUtils = elanUtils;
 197         this.jobCoordinator = jobCoordinator;
 198         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 199         this.elanInstanceCache = elanInstanceCache;
 200         this.elanInterfaceCache = elanInterfaceCache;
 201         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 202     }
 203
 204     @Override
 205     @PostConstruct
 206     public void init() {
 207         registerListener();
 208     }
 209
 210     @Override
 211     public void registerListener() {
 212         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 213     }
 214
 215     @Override
 216     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 217         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 218     }
 219
 220     @Override
 221     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 222         String interfaceName = del.getName();
 223         String elanInstanceName = del.getElanInstanceName();
 224         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 225         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 226             elanInterfaces.remove(del);
 227             if (elanInterfaces.isEmpty()) {
 228                 unProcessedElanInterfaces.remove(elanInstanceName);
 229             }
 230         }
 231         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 232         /*
 233          * Handling in case the elan instance is deleted.If the Elan instance is
 234          * deleted, there is no need to explicitly delete the elan interfaces
 235          */
 236         if (elanInfo == null) {
 237             return;
 238         }
 239         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 240         if (interfaceInfo == null && elanInfo.isExternal()) {
 241             // In deleting external network, the underlying ietf Inteface might have been removed
 242             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 243             // from Operational DS instead
 244             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 245         }
 246         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 247                 interfaceName, interfaceInfo, this);
 248         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 249     }
 250
 251     private static class RemoveElanInterfaceHolder {
 252         boolean isLastElanInterface = false;
 253         boolean isLastInterfaceOnDpn = false;
 254         BigInteger dpId = null;
 255     }
 256
 257     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 258     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 259             InterfaceInfo interfaceInfo) {
 260         String elanName = elanInfo.getElanInstanceName();
 261         long elanTag = elanInfo.getElanTag();
 262         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 263         List<ListenableFuture<Void>> futures = new ArrayList<>();
 264         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 265         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 266             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 267             if (elanState == null) {
 268                 return;
 269             }
 270             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 271                 List<String> elanInterfaces = elanState.getElanInterfaces();
 272                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 273                     holder.isLastElanInterface = true;
 274                 }
 275                 if (interfaceInfo != null) {
 276                     holder.dpId = interfaceInfo.getDpId();
 277                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 278                         interfaceName, elanTag, interfaceTx);
 279                     /*
 280                      * If there are not elan ports, remove the unknown dmac, terminating
 281                      * service table flows, remote/local bc group
 282                      */
 283                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 284                         || dpnInterfaces.getInterfaces().isEmpty()) {
 285                         // No more Elan Interfaces in this DPN
 286                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 287                             holder.dpId);
 288                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 289                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 290                         }
 291                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 292                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 293                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 296                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 297                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 299                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 300                             }
 301                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 302                         }
 303                         holder.isLastInterfaceOnDpn = true;
 304                     } else {
 305                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 306                     }
 307                 }
 308             }));
 309         }));
 310         futures.forEach(ElanUtils::waitForTransactionToComplete);
 311
 312         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 313             futures.add(
 314                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 315                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 316         }
 317         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 318                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 319         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 320                 ElanConstants.JOB_MAX_RETRIES);
 321
 322         return futures;
 323     }
 324
 325     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 326             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 327             throws ExecutionException, InterruptedException {
 328         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 329         if (etreeLeafTag != null) {
 330             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 331             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 332         }
 333     }
 334
 335     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 336             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 337             throws ExecutionException, InterruptedException {
 338         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340     }
 341
 342     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 349             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 350             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 351         }
 352     }
 353
 354     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 358         if (etreeInstance != null) {
 359             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 360             BigInteger dpnId = interfaceInfo.getDpId();
 361             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 362             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 363             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 364         }
 365     }
 366
 367     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 368             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 369         String elanName = elanInfo.getElanInstanceName();
 370         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 371         if (elanState == null) {
 372             return elanState;
 373         }
 374         List<String> elanInterfaces = elanState.getElanInterfaces();
 375         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 376         if (!isRemoved) {
 377             return elanState;
 378         }
 379
 380         if (elanInterfaces.isEmpty()) {
 381             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 384             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 385         } else {
 386             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 387                     .withKey(new ElanKey(elanName)).build();
 388             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 389         }
 390         return elanState;
 391     }
 392
 393     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 394             throws ReadFailedException {
 395         // removing the ElanInterface from the config data_store if interface is
 396         // not present in Interface config DS
 397         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 398             interfaceName) == null
 399                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 400             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 401         }
 402     }
 403
 404     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 405             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 406         String elanName = elanInfo.getElanInstanceName();
 407         List<ListenableFuture<Void>> futures = new ArrayList<>();
 408         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 409             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 410                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 411                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 412                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 413                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 414                 if (interfaceInfo != null) {
 415                     if (existingElanInterfaceMac.isPresent()) {
 416                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 417                         if (existingMacEntries != null) {
 418                             List<PhysAddress> macAddresses = new ArrayList<>();
 419                             for (MacEntry macEntry : existingMacEntries) {
 420                                 PhysAddress macAddress = macEntry.getMacAddress();
 421                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 422                                         macAddress.getValue(), interfaceName);
 423                                 Optional<MacEntry> macEntryOptional =
 424                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 425                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 426                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 427                                 }
 428                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 429                                 macAddresses.add(macAddress);
 430                             }
 431
 432                             // Removing all those MACs from External Devices belonging
 433                             // to this ELAN
 434                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 435                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 436                             }
 437                         }
 438                     }
 439                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 440                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 441                 } else if (existingElanInterfaceMac.isPresent()) {
 442                     // Interface does not exist in ConfigDS, so lets remove everything
 443                     // about that interface related to Elan
 444                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 445                     if (macEntries != null) {
 446                         for (MacEntry macEntry : macEntries) {
 447                             PhysAddress macAddress = macEntry.getMacAddress();
 448                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 449                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 450                             }
 451                         }
 452                     }
 453                 }
 454                 if (existingElanInterfaceMac.isPresent()) {
 455                     interfaceTx.delete(elanInterfaceId);
 456                 }
 457                 unbindService(interfaceName, flowTx);
 458                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 459             }));
 460         }));
 461         return futures;
 462     }
 463
 464     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 465                                                                          String interfaceName, long elanTag,
 466                                                                          TypedReadWriteTransaction<Operational> tx)
 467             throws ExecutionException, InterruptedException {
 468         // FIXME: pass in and use ElanInstanceKey instead?
 469         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 470         lock.lock();
 471         try {
 472             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 473             if (dpnInterfaces != null) {
 474                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 475                 if (interfaceLists != null) {
 476                     interfaceLists.remove(interfaceName);
 477                 }
 478
 479                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 480                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 481                     deleteElanDpnInterface(elanName, dpId, tx);
 482                 } else {
 483                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 484                 }
 485             }
 486             return dpnInterfaces;
 487         } finally {
 488             lock.unlock();
 489         }
 490     }
 491
 492     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 493         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 494         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 495             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 496                 BigInteger currentDpId = dpnInterface.getDpId();
 497                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 498                     for (String elanInterface : dpnInterface.getInterfaces()) {
 499                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 500                         if (macs == null || macs.getMacEntry() == null) {
 501                             continue;
 502                         }
 503                         for (MacEntry mac : macs.getMacEntry()) {
 504                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 505                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 506                         }
 507                     }
 508                 }
 509             }
 510         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 511     }
 512
 513     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 514             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 515         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 516         if (etreeLeafTag != null) {
 517             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 518         }
 519     }
 520
 521     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 522             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 523         mdsalManager
 524                 .removeFlow(confTx, dpId,
 525                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 526                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 527                                         mac.getMacAddress().getValue(), elanTag)));
 528     }
 529
 530     /*
 531     * Possible Scenarios for update
 532     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 533         then all {1,2,3,4} should be removed
 534
 535         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 536         then all {1,2,3,4} should be added
 537
 538         c. if orig = {1,2,3,4} updated={2,3,4}
 539         then 1 should be removed
 540
 541         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 542         then we should just add 5
 543
 544         e. if orig = {1,2,3,4} updated={2,3,4,5}
 545         then 1 should be removed , 5 should be added
 546     * */
 547     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 548     @Override
 549     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 550         // updating the static-Mac Entries for the existing elanInterface
 551         String elanName = update.getElanInstanceName();
 552         String interfaceName = update.getName();
 553
 554         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 555         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 556         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 557         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 558
 559         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 560                 deletedEntry.getMacAddress()));
 561
 562         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 563         * Hence add the macentries for the same.*/
 564         for (StaticMacEntries staticMacEntry : updatedEntries) {
 565             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 566                     staticMacEntry.getMacAddress());
 567             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 568                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 569                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 570                     if (existingMacEntry.isPresent()) {
 571                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 572                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 573                             tx);
 574                     } else {
 575                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 576                             elanName, interfaceName, staticMacEntry, tx);
 577                     }
 578                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 579         }
 580     }
 581
 582     @Override
 583     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 584         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 585             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 586             String interfaceName = elanInterfaceAdded.getName();
 587             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 588             if (interfaceInfo == null) {
 589                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 590                 return;
 591             }
 592             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 593
 594             if (elanInstance == null) {
 595                 // Add the ElanInstance in the Configuration data-store
 596                 List<String> elanInterfaces = new ArrayList<>();
 597                 elanInterfaces.add(interfaceName);
 598                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 599                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 600                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 601                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 602             }
 603
 604             Long elanTag = elanInstance.getElanTag();
 605             // If elan tag is not updated, then put the elan interface into
 606             // unprocessed entry map and entry. Let entries
 607             // in this map get processed during ELAN update DCN.
 608             if (elanTag == null || elanTag == 0L) {
 609                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 610                 if (elanInterfaces == null) {
 611                     elanInterfaces = new ConcurrentLinkedQueue<>();
 612                 }
 613                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 614                     elanInterfaces.add(elanInterfaceAdded);
 615                 }
 616                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 617                 return;
 618             }
 619             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 620                     interfaceInfo, elanInstance, this);
 621             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 622         }), LOG, "Error processing added ELAN interface");
 623     }
 624
 625     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 626         List<ListenableFuture<Void>> futures = new ArrayList<>();
 627         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 628         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 629             return futures;
 630         }
 631         for (ElanInterface elanInterface : elanInterfaces) {
 632             String interfaceName = elanInterface.getName();
 633             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 634             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 635         }
 636         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 637         return futures;
 638     }
 639
 640     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 641             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 642         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 643                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 644         List<DpnInterfaces> dpnInterfaceLists = null;
 645         if (elanDpnInterfacesList != null) {
 646             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 647         }
 648         if (dpnInterfaceLists == null) {
 649             dpnInterfaceLists = new ArrayList<>();
 650         }
 651         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 652             BigInteger dstDpId = interfaceInfo.getDpId();
 653             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 654                 continue;
 655             }
 656             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 657             for (String remoteIf : remoteElanInterfaces) {
 658                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 659                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 660                 if (elanIfMac == null || remoteInterface == null) {
 661                     continue;
 662                 }
 663                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 664                 if (remoteMacEntries != null) {
 665                     for (MacEntry macEntry : remoteMacEntries) {
 666                         String macAddress = macEntry.getMacAddress().getValue();
 667                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 668                                 dstDpId, elanInstance.getElanInstanceName());
 669                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 670                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 671                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 672                     }
 673                 }
 674             }
 675         }
 676     }
 677
 678     private static class AddElanInterfaceHolder {
 679         private DpnInterfaces dpnInterfaces = null;
 680         private boolean isFirstInterfaceInDpn = false;
 681         private BigInteger dpId;
 682     }
 683
 684     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 685     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 686             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 687         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 688         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 689         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 690
 691         String interfaceName = elanInterface.getName();
 692         String elanInstanceName = elanInterface.getElanInstanceName();
 693
 694         List<ListenableFuture<Void>> futures = new ArrayList<>();
 695         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 696         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 697             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 698             if (elanInfo == null) {
 699                 List<String> elanInterfaces = new ArrayList<>();
 700                 elanInterfaces.add(interfaceName);
 701                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 702                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 703                         operTx)));
 704             } else {
 705                 createElanStateList(elanInstanceName, interfaceName, operTx);
 706             }
 707             // Specific actions to the DPN where the ElanInterface has been added,
 708             // for example, programming the
 709             // External tunnel table if needed or adding the ElanInterface to the
 710             // DpnInterfaces in the operational DS.
 711             holder.dpId = interfaceInfo.getDpId();
 712             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 713                 // FIXME: use elanInstaince.key() instead?
 714                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 715                 lock.lock();
 716                 try {
 717                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 718                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 719                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 720                     if (ElanUtils.isVlan(elanInstance)) {
 721                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 722                             elanInstanceName, existingElanDpnInterfaces);
 723                     } else {
 724                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 725                     }
 726                     if (holder.isFirstInterfaceInDpn) {
 727                         // ELAN's 1st ElanInterface added to this DPN
 728                         if (!existingElanDpnInterfaces.isPresent()) {
 729                             holder.dpnInterfaces =
 730                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 731                         } else {
 732                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 733                             List<String> elanInterfaces =
 734                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 735                             elanInterfaces.add(interfaceName);
 736                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 737                                 elanInterfaces, operTx);
 738                         }
 739                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 740                         // table, but only if Elan has VNI
 741                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 742                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 743                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 744                         }
 745                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 746                             interfaceName);
 747                     } else {
 748                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 749                         List<String> elanInterfaces =
 750                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 751                         elanInterfaces.add(interfaceName);
 752                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 753                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 754                                 interfaceName);
 755                         }
 756                         holder.dpnInterfaces =
 757                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 758                     }
 759                 } finally {
 760                     lock.unlock();
 761                 }
 762             }
 763
 764             // add code to install Local/Remote BC group, unknow DMAC entry,
 765             // terminating service table flow entry
 766             // call bindservice of interfacemanager to create ingress table flow
 767             // enty.
 768             // Add interface to the ElanInterfaceForwardingEntires Container
 769             createElanInterfaceTablesList(interfaceName, operTx);
 770         }));
 771         futures.forEach(ElanUtils::waitForTransactionToComplete);
 772         futures.add(
 773             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 774                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 775                     holder.isFirstInterfaceInDpn, confTx))));
 776
 777         // add the vlan provider interface to remote BC group for the elan
 778         // for internal vlan networks
 779         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 780             if (interfaceManager.isExternalInterface(interfaceName)) {
 781                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 782                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 783             }
 784         }
 785
 786         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 787             //update the remote-DPNs remoteBC group entry with Tunnels
 788             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 789                 holder.dpId);
 790             futures.add(
 791                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 792                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 793         }
 794
 795         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 796         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 797                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 798         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 799         return futures;
 800     }
 801
 802     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 803     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 804             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 805         String elanInstanceName = elanInstance.getElanInstanceName();
 806         String interfaceName = elanInterface.getName();
 807         List<ListenableFuture<Void>> futures = new ArrayList<>();
 808         BigInteger dpId = interfaceInfo.getDpId();
 809         boolean isInterfaceOperational = isOperational(interfaceInfo);
 810         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 811             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 812                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 813                     isFirstInterfaceInDpn, confTx);
 814
 815                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 816                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 817
 818                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 819                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 820                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 821                             staticMacEntry.getMacAddress());
 822                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 823                             LogicalDatastoreType.OPERATIONAL, macId);
 824                         if (existingMacEntry.isPresent()) {
 825                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 826                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 827                                 existingMacEntry.get(), operTx);
 828                         } else {
 829                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 830                                 interfaceName, staticMacEntry, operTx);
 831                         }
 832
 833                         if (isInterfaceOperational) {
 834                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 835                             // DPNs
 836                             String macAddress = staticMacEntry.getMacAddress().getValue();
 837                             LOG.info(
 838                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 839                                     + " {}",
 840                                 macAddress, elanInstanceName, interfaceName);
 841                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 842                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 843                         }
 844                     }
 845
 846                     if (isInterfaceOperational) {
 847                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 848                         // on purpose.
 849                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 850                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 851                         }
 852                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 853                             staticMacAddresses);
 854                     }
 855                 }
 856             }));
 857         }));
 858         futures.forEach(ElanUtils::waitForTransactionToComplete);
 859         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 860             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 861             try {
 862                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 863                 if (neutronPort != null) {
 864                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 865                 }
 866             } catch (IllegalArgumentException ex) {
 867                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 868             }
 869         }
 870         return futures;
 871     }
 872
 873     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 874             PhysAddress physAddress) {
 875         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 876         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 877         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 878                 LogicalDatastoreType.OPERATIONAL, macId);
 879
 880         if (!existingMacEntry.isPresent()) {
 881             return;
 882         }
 883
 884         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 885                 .withKey(new MacEntryKey(physAddress)).build();
 886         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 887                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 888     }
 889
 890     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 891             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 892         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 893         if (!existingElanDpnInterfaces.isPresent()) {
 894             return true;
 895         }
 896         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 897             return false;
 898         }
 899         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 900         int dummyInterfaceCount =  0;
 901         List<String> interfaces = dpnInterfaces.getInterfaces();
 902         if (interfaces == null) {
 903             return true;
 904         }
 905         if (interfaces.contains(routerPortUuid)) {
 906             dummyInterfaceCount++;
 907         }
 908         if (interfaces.contains(elanInstanceName)) {
 909             dummyInterfaceCount++;
 910         }
 911         return interfaces.size() == dummyInterfaceCount;
 912     }
 913
 914     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 915         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 916                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 917     }
 918
 919     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 920             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 921         if (!isOperational(interfaceInfo)) {
 922             return;
 923         }
 924         BigInteger dpId = interfaceInfo.getDpId();
 925         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 926             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 927         }
 928         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 929         if (isFirstInterfaceInDpn) {
 930             // Terminating Service , UnknownDMAC Table.
 931             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 932             // for ELAN Instance is VxLAN
 933             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 934                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 935             }
 936             setupUnknownDMacTable(elanInstance, dpId, confTx);
 937             /*
 938              * Install remote DMAC flow. This is required since this DPN is
 939              * added later to the elan instance and remote DMACs of other
 940              * interfaces in this elan instance are not present in the current
 941              * dpn.
 942              */
 943             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 944                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 945                         elanInstance.getElanInstanceName());
 946                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 947             }
 948         }
 949         // bind the Elan service to the Interface
 950         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 951     }
 952
 953     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 954             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 955         if (!isOperational(interfaceInfo)) {
 956             return;
 957         }
 958         // LocalBroadcast Group creation with elan-Interfaces
 959         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 960         if (isFirstInterfaceInDpn) {
 961             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 962             BigInteger dpId = interfaceInfo.getDpId();
 963             // RemoteBroadcast Group creation
 964             try {
 965                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 966             } catch (InterruptedException e1) {
 967                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 968             }
 969             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 970             try {
 971                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 972             } catch (InterruptedException e1) {
 973                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 974             }
 975         }
 976     }
 977
 978     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 979             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 980         int ifTag = interfaceInfo.getInterfaceTag();
 981         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 982                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 983                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 984                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 985                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 986
 987         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 988
 989         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 990                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 991                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 992                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 993
 994         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 995     }
 996
 997     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 998             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 999         int ifTag = interfaceInfo.getInterfaceTag();
1000         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1001             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1002
1003         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1004
1005         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1006             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1007             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1008             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1009
1010         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1011     }
1012
1013     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1014             TypedWriteTransaction<Configuration> confTx) {
1015         int elanTag = elanInfo.getElanTag().intValue();
1016         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1017         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1018         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1019         if (etreeInstance != null) {
1020             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1021             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1022             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1023         }
1024     }
1025
1026     @SuppressWarnings("checkstyle:IllegalCatch")
1027     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1028             TypedWriteTransaction<Configuration> confTx) {
1029         int bucketId = 0;
1030         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1031         if (elanDpns != null) {
1032             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1033             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1034                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1035                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1036                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1037                     List<Action> listAction = new ArrayList<>();
1038                     int actionKey = 0;
1039                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1040                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1041                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1042                     bucketId++;
1043                     for (DpnInterfaces otherFes : dpnInterfaces) {
1044                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1045                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1046                             && !otherFes.getInterfaces().isEmpty()) {
1047                             try {
1048                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1049                                         dpnInterface.getDpId(), otherFes.getDpId(),
1050                                         elanUtils.isOpenstackVniSemanticsEnforced()
1051                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1052                                 if (!remoteListActionInfo.isEmpty()) {
1053                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1054                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1055                                     bucketId++;
1056                                 }
1057                             } catch (Exception ex) {
1058                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1059                                         + "Logical Group Interface not found between source Dpn - {}, "
1060                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1061                                 return;
1062                             }
1063                         }
1064                     }
1065                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1066                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1067                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1068
1069                     if (remoteListBucketInfo.isEmpty()) {
1070                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1071                         continue;
1072                     }
1073                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1074                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1075                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1076                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1077                 }
1078             }
1079             try {
1080                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1081             } catch (InterruptedException e1) {
1082                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1083             }
1084         }
1085     }
1086
1087     private List<MatchInfo> buildMatchesForVni(Long vni) {
1088         List<MatchInfo> mkMatches = new ArrayList<>();
1089         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1090         mkMatches.add(match);
1091         return mkMatches;
1092     }
1093
1094     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1095         List<InstructionInfo> mkInstructions = new ArrayList<>();
1096         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1097         return mkInstructions;
1098     }
1099
1100     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1101         List<MatchInfo> mkMatches = new ArrayList<>();
1102         // Matching metadata
1103         mkMatches.add(new MatchMetadata(
1104                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1105         return mkMatches;
1106     }
1107
1108     /**
1109      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1110      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1111      *
1112      * @param elanTag
1113      *            elanTag to be written in metadata when flow is selected
1114      * @return the instructions ready to be installed in a flow
1115      */
1116     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1117         List<InstructionInfo> mkInstructions = new ArrayList<>();
1118         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1119                 .getElanMetadataMask()));
1120         /* applicable for EXTERNAL_TUNNEL_TABLE only
1121         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1122         */
1123         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1124         return mkInstructions;
1125     }
1126
1127     // Install DMAC entry on dst DPN
1128     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1129             BigInteger dstDpId) {
1130         String interfaceName = interfaceInfo.getInterfaceName();
1131         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1132         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1133             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1134             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1135                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1136                     for (MacEntry macEntry : macEntries) {
1137                         String macAddress = macEntry.getMacAddress().getValue();
1138                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1139                             interfaceName);
1140                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1141                             interfaceInfo.getDpId())) {
1142                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1143                                 macAddress);
1144                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1145                         }
1146                     }
1147                 })));
1148         }
1149         return emptyList();
1150     }
1151
1152     private void createDropBucket(List<Bucket> listBucket) {
1153         List<Action> actionsInfos = new ArrayList<>();
1154         actionsInfos.add(new ActionDrop().buildAction());
1155         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1156                 MDSALUtil.WATCH_GROUP);
1157         listBucket.add(dropBucket);
1158     }
1159
1160     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1161             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1162         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1163         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1164     }
1165
1166     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1167             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1168         List<Bucket> listBucket = new ArrayList<>();
1169         int bucketId = 0;
1170         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1171
1172         List<String> interfaces = new ArrayList<>();
1173         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1174             interfaces = newDpnInterface.getInterfaces();
1175         }
1176         for (String ifName : interfaces) {
1177             // In case if there is a InterfacePort in the cache which is not in
1178             // operational state, skip processing it
1179             InterfaceInfo ifInfo = interfaceManager
1180                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1181             if (!isOperational(ifInfo)) {
1182                 continue;
1183             }
1184
1185             if (!interfaceManager.isExternalInterface(ifName)) {
1186                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1187                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1188                 bucketId++;
1189             }
1190         }
1191
1192         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1193                 MDSALUtil.buildBucketLists(listBucket));
1194         LOG.trace("installing the localBroadCast Group:{}", group);
1195         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1196     }
1197
1198     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1199             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1200         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1201         if (etreeInstance != null) {
1202             List<Bucket> listBucket = new ArrayList<>();
1203             int bucketId = 0;
1204
1205             List<String> interfaces = new ArrayList<>();
1206             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1207                 interfaces = newDpnInterface.getInterfaces();
1208             }
1209             for (String ifName : interfaces) {
1210                 // In case if there is a InterfacePort in the cache which is not
1211                 // in
1212                 // operational state, skip processing it
1213                 InterfaceInfo ifInfo = interfaceManager
1214                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1215                 if (!isOperational(ifInfo)) {
1216                     continue;
1217                 }
1218
1219                 if (!interfaceManager.isExternalInterface(ifName)) {
1220                     // only add root interfaces
1221                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1222                 }
1223             }
1224
1225             if (listBucket.isEmpty()) { // No Buckets
1226                 createDropBucket(listBucket);
1227             }
1228
1229             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1230             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1231             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1232                     MDSALUtil.buildBucketLists(listBucket));
1233             LOG.trace("installing the localBroadCast Group:{}", group);
1234             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1235         }
1236     }
1237
1238     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1239             InterfaceInfo ifInfo) {
1240         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1241         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1242             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1243                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1244             bucketId++;
1245         }
1246         return bucketId;
1247     }
1248
1249     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1250             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1251             throws ExecutionException, InterruptedException {
1252         BigInteger dpnId = interfaceInfo.getDpId();
1253         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1254         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1255         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1256     }
1257
1258     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1259             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1260             throws ExecutionException, InterruptedException {
1261         BigInteger dpnId = interfaceInfo.getDpId();
1262         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1263         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1264         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1265     }
1266
1267     /**
1268      * Installs a flow in the External Tunnel table consisting in translating
1269      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1270      * elanTag that will be used later in the ELANs pipeline.
1271      * @param dpnId the dpn id
1272      */
1273     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1274             TypedWriteTransaction<Configuration> confTx) {
1275         long elanTag = elanInfo.getElanTag();
1276         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1277                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1278                 elanInfo.getElanInstanceName(), // flowName
1279                 0, // idleTimeout
1280                 0, // hardTimeout
1281                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1282                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1283                 getInstructionsIntOrExtTunnelTable(elanTag));
1284
1285         mdsalManager.addFlow(confTx, flowEntity);
1286     }
1287
1288     /**
1289      * Removes, from External Tunnel table, the flow that translates from VNI to
1290      * elanTag. Important: ensure this method is only called whenever there is
1291      * no other ElanInterface in the specified DPN
1292      *
1293      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1294      */
1295     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1296             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1297         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1298         // last ElanInstance plus
1299         // adding a new one does (almost at the same time) are executed in that
1300         // exact order
1301
1302         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1303         FlowEntity flowEntity = new FlowEntityBuilder()
1304             .setDpnId(dpnId)
1305             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1306             .setFlowId(flowId)
1307             .build();
1308         mdsalManager.removeFlow(confTx, flowEntity);
1309     }
1310
1311     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1312             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1313         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1314         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1315     }
1316
1317     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1318             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1319         List<? extends MatchInfoBase> listMatchInfoBase;
1320         List<InstructionInfo> instructionInfos;
1321         long serviceId;
1322         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1323             serviceId = elanTag;
1324             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1325             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1326         } else {
1327             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1328             listMatchInfoBase = buildMatchesForVni(serviceId);
1329             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1330         }
1331         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1332                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1333                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1334                 instructionInfos);
1335         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1336     }
1337
1338     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1339             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1340         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1341         if (etreeInstance != null) {
1342             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1343         }
1344     }
1345
1346     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1347             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1348         long elanTag = elanInfo.getElanTag();
1349         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1350         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1351         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1352     }
1353
1354     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1355             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1356         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1357         if (etreeLeafTag != null) {
1358             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1359             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1360             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1361         }
1362     }
1363
1364     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1365             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1366         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1367                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1368                 5, elanInfo.getElanInstanceName(), 0, 0,
1369                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1370                 getMatchesForElanTag(elanTag, /* SH flag */false),
1371                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1372
1373         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1374     }
1375
1376     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1377             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1378         // only if ELAN can connect to external network, perform the following
1379
1380         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1381             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1382                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1383                     5, elanInfo.getElanInstanceName(), 0, 0,
1384                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1385                     getMatchesForElanTag(elanTag, /* SH flag */true),
1386                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1387             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1388         }
1389     }
1390
1391
1392     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1393             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1394             throws ExecutionException, InterruptedException {
1395         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1396                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1397         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1398
1399         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1400             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1401                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1402                     .build();
1403             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1404         }
1405     }
1406
1407     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1408         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1409     }
1410
1411     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1412             TypedWriteTransaction<Configuration> tx) {
1413         if (isStandardElanService(elanInterface)) {
1414             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1415                     elanInterface.getName(), lportTag, tx);
1416         } else { // Etree service
1417             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1418         }
1419     }
1420
1421     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1422             TypedWriteTransaction<Configuration> tx) {
1423         int instructionKey = 0;
1424         List<Instruction> instructions = new ArrayList<>();
1425         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1426                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1427
1428         List<Action> actions = new ArrayList<>();
1429         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1430                 lportTag).buildAction());
1431         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1432                 elanTag).buildAction());
1433         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1434
1435         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1436                 ++instructionKey));
1437
1438         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1439         BoundServices serviceInfo = ElanUtils.getBoundServices(
1440                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1441                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1442         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1443         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1444                 bindServiceId);
1445         if (!existingElanService.isPresent()) {
1446             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1447         }
1448     }
1449
1450     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1451             TypedWriteTransaction<Configuration> tx) {
1452         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1453             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1454                     lportTag, tx);
1455         } else {
1456             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1457             if (etreeInstance == null) {
1458                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1459                         elanInterface.getName(), elanInfo.getElanInstanceName());
1460             } else {
1461                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1462                         elanInterface.getName(), lportTag, tx);
1463             }
1464         }
1465     }
1466
1467     private boolean isStandardElanService(ElanInterface elanInterface) {
1468         return elanInterface.augmentation(EtreeInterface.class) == null;
1469     }
1470
1471     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1472             throws ExecutionException, InterruptedException {
1473         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1474         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1475         if (tx.read(bindServiceId).get().isPresent()) {
1476             tx.delete(bindServiceId);
1477         }
1478     }
1479
1480     private String getFlowRef(long tableId, long elanTag) {
1481         return String.valueOf(tableId) + elanTag;
1482     }
1483
1484     private String getFlowRef(long tableId, long elanTag, String flowName) {
1485         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1486                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1487     }
1488
1489     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1490         return String.valueOf(tableId) + elanTag + shFlag;
1491     }
1492
1493     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1494         List<Action> listAction = new ArrayList<>();
1495         int actionKey = 0;
1496         listAction.add(
1497             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1498         actionKey++;
1499         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1500         return listAction;
1501     }
1502
1503     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1504             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1505         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1506                 .withKey(new DpnInterfacesKey(dpId)).build();
1507         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1508                 CREATE_MISSING_PARENTS);
1509         return dpnInterface;
1510     }
1511
1512     /**
1513      * Delete elan dpn interface from operational DS.
1514      *
1515      * @param elanInstanceName
1516      *            the elan instance name
1517      * @param dpId
1518      *            the dp id
1519      */
1520     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1521             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1522         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1523                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1524         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1525         if (dpnInterfaces.isPresent()) {
1526             tx.delete(dpnInterfacesId);
1527         }
1528     }
1529
1530     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1531             TypedWriteTransaction<Operational> tx) {
1532         List<String> interfaceNames = new ArrayList<>();
1533         interfaceNames.add(interfaceName);
1534         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1535                 .withKey(new DpnInterfacesKey(dpId)).build();
1536         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1537                 CREATE_MISSING_PARENTS);
1538         return dpnInterface;
1539     }
1540
1541     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1542             throws ExecutionException, InterruptedException {
1543         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1544                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1545         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1546         // Adding new Elan Interface Port to the operational DataStore without
1547         // Static-Mac Entries..
1548         if (!interfaceMacTables.isPresent()) {
1549             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1550                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1551             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1552                     CREATE_MISSING_PARENTS);
1553         }
1554     }
1555
1556     private void createElanStateList(String elanInstanceName, String interfaceName,
1557             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1558         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1559         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1560         // Adding new Elan Interface Port to the operational DataStore without
1561         // Static-Mac Entries..
1562         if (elanInterfaceLists.isPresent()) {
1563             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1564             if (interfaceLists == null) {
1565                 interfaceLists = new ArrayList<>();
1566             }
1567             interfaceLists.add(interfaceName);
1568             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1569                     .withKey(new ElanKey(elanInstanceName)).build();
1570             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1571         }
1572     }
1573
1574     private boolean isOperational(InterfaceInfo interfaceInfo) {
1575         if (interfaceInfo == null) {
1576             return false;
1577         }
1578         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1579     }
1580
1581     @SuppressWarnings("checkstyle:IllegalCatch")
1582     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1583         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1584         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1585                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1586         if (dpnInterfaceLists == null) {
1587             return;
1588         }
1589         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1590         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1591             int cnt = 0;
1592             String elanName = elanDpns.getElanInstanceName();
1593             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1594             if (elanInfo == null) {
1595                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1596                         + "skipping this ELAN for tunnel handling", elanName);
1597                 continue;
1598             }
1599             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1600                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1601                 continue;
1602             }
1603             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1604             if (dpnInterfaces == null) {
1605                 continue;
1606             }
1607             DpnInterfaces dstDpnIf = null;
1608             for (DpnInterfaces dpnIf : dpnInterfaces) {
1609                 BigInteger dpnIfDpId = dpnIf.getDpId();
1610                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1611                     cnt++;
1612                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1613                     cnt++;
1614                     dstDpnIf = dpnIf;
1615                 }
1616             }
1617             if (cnt == 2) {
1618                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1619                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1620                 jobCoordinator.enqueueJob(elanName, () -> {
1621                     // update Remote BC Group
1622                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1623                     try {
1624                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1625                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1626                                 confTx)).get();
1627                     } catch (RuntimeException e) {
1628                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1629                     }
1630                     Set<String> interfaceLists = new HashSet<>();
1631                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1632                     for (String ifName : interfaceLists) {
1633                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1634                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1635                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1636                             if (isOperational(interfaceInfo)) {
1637                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1638                             }
1639                             return emptyList();
1640                         }, ElanConstants.JOB_MAX_RETRIES);
1641                     }
1642                     return emptyList();
1643                 }, ElanConstants.JOB_MAX_RETRIES);
1644             }
1645
1646         }
1647     }
1648
1649     /**
1650      * Handle external tunnel state event.
1651      *
1652      * @param externalTunnel
1653      *            the external tunnel
1654      * @param intrf
1655      *            the interface
1656      */
1657     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1658         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1659             return;
1660         }
1661         // dpId/externalNodeId will be available either in source or destination
1662         // based on the tunnel end point
1663         BigInteger dpId = null;
1664         NodeId externalNodeId = null;
1665         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1666             dpId = new BigInteger(externalTunnel.getSourceDevice());
1667             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1668         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1669             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1670             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1671         }
1672         if (dpId == null || externalNodeId == null) {
1673             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1674             return;
1675         }
1676
1677         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1678         if (dpnInterfaceLists == null) {
1679             return;
1680         }
1681         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1682         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1683             String elanName = elanDpns.getElanInstanceName();
1684             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1685
1686             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1687             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1688                     || dpnInterfaces.getInterfaces().isEmpty()) {
1689                 continue;
1690             }
1691             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1692
1693             final BigInteger finalDpId = dpId;
1694             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1695                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1696                 "Error setting up ELAN BGs");
1697             // install L2gwDevices local macs in dpn.
1698             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1699             // Install dpn macs on external device
1700             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1701                     externalNodeId);
1702         }
1703         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1704     }
1705
1706     /**
1707      * Installs dpn macs in external device. first it checks if the physical
1708      * locator towards this dpn tep is present or not if the physical locator is
1709      * present go ahead and add the ucast macs otherwise update the mcast mac
1710      * entry to include this dpn tep ip and schedule the job to put ucast macs
1711      * once the physical locator is programmed in device
1712      *
1713      * @param elanName
1714      *            the elan name
1715      * @param lstElanInterfaceNames
1716      *            the lst Elan interface names
1717      * @param dpnId
1718      *            the dpn id
1719      * @param externalNodeId
1720      *            the external node id
1721      */
1722     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1723             NodeId externalNodeId) {
1724         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1725                 externalNodeId.getValue());
1726         if (elanL2GwDevice == null) {
1727             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1728             return;
1729         }
1730         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1731         if (dpnTepIp == null) {
1732             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1733                     externalNodeId);
1734             return;
1735         }
1736
1737         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1738         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1739                 LogicalDatastoreType.OPERATIONAL);
1740         boolean phyLocAlreadyExists =
1741                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1742                 dpnTepIp);
1743         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1744                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1745         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1746
1747         if (phyLocAlreadyExists) {
1748             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1749             return;
1750         }
1751         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1752         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1753     }
1754
1755     /**
1756      * Validate external tunnel state event.
1757      *
1758      * @param externalTunnel
1759      *            the external tunnel
1760      * @param intrf
1761      *            the intrf
1762      * @return true, if successful
1763      */
1764     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1765         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1766             String srcDevice = externalTunnel.getDestinationDevice();
1767             String destDevice = externalTunnel.getSourceDevice();
1768             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1769                     LogicalDatastoreType.CONFIGURATION);
1770             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1771                     otherEndPointExtTunnel);
1772             if (otherEndPointExtTunnel != null) {
1773                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1774                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1775                 if (otherEndPointInterfaceOperational) {
1776                     return true;
1777                 } else {
1778                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1779                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1780                 }
1781             }
1782         }
1783         return false;
1784     }
1785
1786     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1787         List<MatchInfo> mkMatches = new ArrayList<>();
1788         // Matching metadata
1789         mkMatches.add(
1790                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1791         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1792         return mkMatches;
1793     }
1794
1795     @Override
1796     protected ElanInterfaceManager getDataTreeChangeListener() {
1797         return this;
1798     }
1799
1800     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1801                                              BigInteger dpId) {
1802         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1803         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1804             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1805                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1806         try {
1807             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1808         } catch (InterruptedException e) {
1809             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1810         }
1811     }
1812 }