elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 135 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 /**
 141  * Class in charge of handling creations, modifications and removals of
 142  * ElanInterfaces.
 143  *
 144  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 145  */
 146 @Singleton
 147 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 148         implements RecoverableListener {
 149     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final boolean SH_FLAG_SET = true;
 152     private static final boolean SH_FLAG_UNSET = false;
 153
 154     private final DataBroker broker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalManager;
 157     private final IInterfaceManager interfaceManager;
 158     private final IdManagerService idManager;
 159     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 160     private final INeutronVpnManager neutronVpnManager;
 161     private final ElanItmUtils elanItmUtils;
 162     private final ElanEtreeUtils elanEtreeUtils;
 163     private final ElanL2GatewayUtils elanL2GatewayUtils;
 164     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 165     private final ElanUtils elanUtils;
 166     private final JobCoordinator jobCoordinator;
 167     private final ElanInstanceCache elanInstanceCache;
 168     private final ElanInterfaceCache elanInterfaceCache;
 169
 170     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 171         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 172
 173     @Inject
 174     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 175                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 176                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 177                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 178                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 179                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 180                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 181                                 final ElanInstanceCache elanInstanceCache,
 182                                 final ElanInterfaceCache elanInterfaceCache,
 183                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 184                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 185         super(ElanInterface.class, ElanInterfaceManager.class);
 186         this.broker = dataBroker;
 187         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 188         this.idManager = managerService;
 189         this.mdsalManager = mdsalApiManager;
 190         this.interfaceManager = interfaceManager;
 191         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 192         this.neutronVpnManager = neutronVpnManager;
 193         this.elanItmUtils = elanItmUtils;
 194         this.elanEtreeUtils = elanEtreeUtils;
 195         this.elanL2GatewayUtils = elanL2GatewayUtils;
 196         this.elanUtils = elanUtils;
 197         this.jobCoordinator = jobCoordinator;
 198         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 199         this.elanInstanceCache = elanInstanceCache;
 200         this.elanInterfaceCache = elanInterfaceCache;
 201         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 202     }
 203
 204     @Override
 205     @PostConstruct
 206     public void init() {
 207         registerListener();
 208     }
 209
 210     @Override
 211     public void registerListener() {
 212         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 213     }
 214
 215     @Override
 216     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 217         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 218     }
 219
 220     @Override
 221     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 222         String interfaceName = del.getName();
 223         String elanInstanceName = del.getElanInstanceName();
 224         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 225         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 226             elanInterfaces.remove(del);
 227             if (elanInterfaces.isEmpty()) {
 228                 unProcessedElanInterfaces.remove(elanInstanceName);
 229             }
 230         }
 231         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 232         /*
 233          * Handling in case the elan instance is deleted.If the Elan instance is
 234          * deleted, there is no need to explicitly delete the elan interfaces
 235          */
 236         if (elanInfo == null) {
 237             return;
 238         }
 239         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 240         if (interfaceInfo == null && elanInfo.isExternal()) {
 241             // In deleting external network, the underlying ietf Inteface might have been removed
 242             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 243             // from Operational DS instead
 244             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 245         }
 246         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 247                 interfaceName, interfaceInfo, this);
 248         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 249     }
 250
 251     private static class RemoveElanInterfaceHolder {
 252         boolean isLastElanInterface = false;
 253         boolean isLastInterfaceOnDpn = false;
 254         BigInteger dpId = null;
 255     }
 256
 257     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 258     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 259             InterfaceInfo interfaceInfo) {
 260         String elanName = elanInfo.getElanInstanceName();
 261         long elanTag = elanInfo.getElanTag();
 262         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 263         List<ListenableFuture<Void>> futures = new ArrayList<>();
 264         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 265         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 266             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 267             if (elanState == null) {
 268                 return;
 269             }
 270             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 271                 List<String> elanInterfaces = elanState.getElanInterfaces();
 272                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 273                     holder.isLastElanInterface = true;
 274                 }
 275                 if (interfaceInfo != null) {
 276                     holder.dpId = interfaceInfo.getDpId();
 277                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 278                         interfaceName, elanTag, interfaceTx);
 279                     /*
 280                      * If there are not elan ports, remove the unknown dmac, terminating
 281                      * service table flows, remote/local bc group
 282                      */
 283                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 284                         || dpnInterfaces.getInterfaces().isEmpty()) {
 285                         // No more Elan Interfaces in this DPN
 286                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 287                             holder.dpId);
 288                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 289                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 290                         }
 291                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 292                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 293                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 296                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 297                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 299                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 300                             }
 301                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 302                         }
 303                         holder.isLastInterfaceOnDpn = true;
 304                     } else {
 305                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 306                     }
 307                 }
 308             }));
 309         }));
 310         futures.forEach(ElanUtils::waitForTransactionToComplete);
 311
 312         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 313             futures.add(
 314                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 315                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 316         }
 317         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 318                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 319         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 320                 ElanConstants.JOB_MAX_RETRIES);
 321
 322         return futures;
 323     }
 324
 325     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 326             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 327             throws ExecutionException, InterruptedException {
 328         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 329         if (etreeLeafTag != null) {
 330             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 331             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 332         }
 333     }
 334
 335     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 336             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 337             throws ExecutionException, InterruptedException {
 338         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340     }
 341
 342     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 349             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 350             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 351         }
 352     }
 353
 354     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 358         if (etreeInstance != null) {
 359             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 360             BigInteger dpnId = interfaceInfo.getDpId();
 361             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 362             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 363             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 364         }
 365     }
 366
 367     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 368             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 369         String elanName = elanInfo.getElanInstanceName();
 370         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 371         if (elanState == null) {
 372             return elanState;
 373         }
 374         List<String> elanInterfaces = elanState.getElanInterfaces();
 375         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 376         if (!isRemoved) {
 377             return elanState;
 378         }
 379
 380         if (elanInterfaces.isEmpty()) {
 381             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 384             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 385         } else {
 386             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 387                     .withKey(new ElanKey(elanName)).build();
 388             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 389         }
 390         return elanState;
 391     }
 392
 393     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 394             throws ReadFailedException {
 395         // removing the ElanInterface from the config data_store if interface is
 396         // not present in Interface config DS
 397         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 398             interfaceName) == null
 399                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 400             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 401         }
 402     }
 403
 404     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 405             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 406         String elanName = elanInfo.getElanInstanceName();
 407         List<ListenableFuture<Void>> futures = new ArrayList<>();
 408         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 409             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 410                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 411                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 412                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 413                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 414                 if (interfaceInfo != null) {
 415                     if (existingElanInterfaceMac.isPresent()) {
 416                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 417                         if (existingMacEntries != null) {
 418                             List<PhysAddress> macAddresses = new ArrayList<>();
 419                             for (MacEntry macEntry : existingMacEntries) {
 420                                 PhysAddress macAddress = macEntry.getMacAddress();
 421                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 422                                         macAddress.getValue(), interfaceName);
 423                                 Optional<MacEntry> macEntryOptional =
 424                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 425                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 426                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 427                                 }
 428                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 429                                 macAddresses.add(macAddress);
 430                             }
 431
 432                             // Removing all those MACs from External Devices belonging
 433                             // to this ELAN
 434                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 435                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 436                             }
 437                         }
 438                     }
 439                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 440                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 441                 } else if (existingElanInterfaceMac.isPresent()) {
 442                     // Interface does not exist in ConfigDS, so lets remove everything
 443                     // about that interface related to Elan
 444                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 445                     if (macEntries != null) {
 446                         for (MacEntry macEntry : macEntries) {
 447                             PhysAddress macAddress = macEntry.getMacAddress();
 448                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 449                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 450                             }
 451                         }
 452                     }
 453                 }
 454                 if (existingElanInterfaceMac.isPresent()) {
 455                     interfaceTx.delete(elanInterfaceId);
 456                 }
 457                 unbindService(interfaceName, flowTx);
 458                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 459             }));
 460         }));
 461         return futures;
 462     }
 463
 464     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 465                                                                          String interfaceName, long elanTag,
 466                                                                          TypedReadWriteTransaction<Operational> tx)
 467             throws ExecutionException, InterruptedException {
 468         // FIXME: pass in and use ElanInstanceKey instead?
 469         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 470         lock.lock();
 471         try {
 472             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 473             if (dpnInterfaces != null) {
 474                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 475                 if (interfaceLists != null) {
 476                     interfaceLists.remove(interfaceName);
 477                 }
 478
 479                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 480                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 481                     deleteElanDpnInterface(elanName, dpId, tx);
 482                 } else {
 483                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 484                 }
 485             }
 486             return dpnInterfaces;
 487         } finally {
 488             lock.unlock();
 489         }
 490     }
 491
 492     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 493         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 494         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 495             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 496                 BigInteger currentDpId = dpnInterface.getDpId();
 497                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 498                     for (String elanInterface : dpnInterface.getInterfaces()) {
 499                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 500                         if (macs == null || macs.getMacEntry() == null) {
 501                             continue;
 502                         }
 503                         for (MacEntry mac : macs.getMacEntry()) {
 504                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 505                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 506                         }
 507                     }
 508                 }
 509             }
 510         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 511     }
 512
 513     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 514             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 515         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 516         if (etreeLeafTag != null) {
 517             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 518         }
 519     }
 520
 521     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 522             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 523         mdsalManager
 524                 .removeFlow(confTx, dpId,
 525                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 526                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 527                                         mac.getMacAddress().getValue(), elanTag)));
 528     }
 529
 530     /*
 531     * Possible Scenarios for update
 532     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 533         then all {1,2,3,4} should be removed
 534
 535         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 536         then all {1,2,3,4} should be added
 537
 538         c. if orig = {1,2,3,4} updated={2,3,4}
 539         then 1 should be removed
 540
 541         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 542         then we should just add 5
 543
 544         e. if orig = {1,2,3,4} updated={2,3,4,5}
 545         then 1 should be removed , 5 should be added
 546     * */
 547     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 548     @Override
 549     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 550         // updating the static-Mac Entries for the existing elanInterface
 551         String elanName = update.getElanInstanceName();
 552         String interfaceName = update.getName();
 553
 554         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 555         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 556         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 557         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 558
 559         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 560                 deletedEntry.getMacAddress()));
 561
 562         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 563         * Hence add the macentries for the same.*/
 564         for (StaticMacEntries staticMacEntry : updatedEntries) {
 565             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 566                     staticMacEntry.getMacAddress());
 567             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 568                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 569                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 570                     if (existingMacEntry.isPresent()) {
 571                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 572                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 573                             tx);
 574                     } else {
 575                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 576                             elanName, interfaceName, staticMacEntry, tx);
 577                     }
 578                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 579         }
 580     }
 581
 582     @Override
 583     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 584         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 585             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 586             String interfaceName = elanInterfaceAdded.getName();
 587             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 588             if (interfaceInfo == null) {
 589                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 590                 return;
 591             }
 592             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 593
 594             if (elanInstance == null) {
 595                 // Add the ElanInstance in the Configuration data-store
 596                 List<String> elanInterfaces = new ArrayList<>();
 597                 elanInterfaces.add(interfaceName);
 598                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 599                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 600                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 601                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 602             }
 603
 604             Long elanTag = elanInstance.getElanTag();
 605             // If elan tag is not updated, then put the elan interface into
 606             // unprocessed entry map and entry. Let entries
 607             // in this map get processed during ELAN update DCN.
 608             if (elanTag == null || elanTag == 0L) {
 609                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 610                 if (elanInterfaces == null) {
 611                     elanInterfaces = new ConcurrentLinkedQueue<>();
 612                 }
 613                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 614                     elanInterfaces.add(elanInterfaceAdded);
 615                 }
 616                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 617                 return;
 618             }
 619             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 620                     interfaceInfo, elanInstance, this);
 621             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 622         }), LOG, "Error processing added ELAN interface");
 623     }
 624
 625     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 626         List<ListenableFuture<Void>> futures = new ArrayList<>();
 627         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 628         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 629             return futures;
 630         }
 631         for (ElanInterface elanInterface : elanInterfaces) {
 632             String interfaceName = elanInterface.getName();
 633             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 634             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 635         }
 636         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 637         return futures;
 638     }
 639
 640     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 641             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 642         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 643                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 644         List<DpnInterfaces> dpnInterfaceLists = null;
 645         if (elanDpnInterfacesList != null) {
 646             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 647         }
 648         if (dpnInterfaceLists == null) {
 649             dpnInterfaceLists = new ArrayList<>();
 650         }
 651         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 652             BigInteger dstDpId = interfaceInfo.getDpId();
 653             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 654                 continue;
 655             }
 656             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 657             for (String remoteIf : remoteElanInterfaces) {
 658                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 659                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 660                 if (elanIfMac == null || remoteInterface == null) {
 661                     continue;
 662                 }
 663                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 664                 if (remoteMacEntries != null) {
 665                     for (MacEntry macEntry : remoteMacEntries) {
 666                         String macAddress = macEntry.getMacAddress().getValue();
 667                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 668                                 dstDpId, elanInstance.getElanInstanceName());
 669                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 670                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 671                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 672                     }
 673                 }
 674             }
 675         }
 676     }
 677
 678     private static class AddElanInterfaceHolder {
 679         private DpnInterfaces dpnInterfaces = null;
 680         private boolean isFirstInterfaceInDpn = false;
 681         private BigInteger dpId;
 682     }
 683
 684     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 685     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 686             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 687         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 688         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 689         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 690
 691         String interfaceName = elanInterface.getName();
 692         String elanInstanceName = elanInterface.getElanInstanceName();
 693
 694         List<ListenableFuture<Void>> futures = new ArrayList<>();
 695         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 696         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 697             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 698             if (elanInfo == null) {
 699                 List<String> elanInterfaces = new ArrayList<>();
 700                 elanInterfaces.add(interfaceName);
 701                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 702                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 703                         operTx)));
 704             } else {
 705                 createElanStateList(elanInstanceName, interfaceName, operTx);
 706             }
 707             // Specific actions to the DPN where the ElanInterface has been added,
 708             // for example, programming the
 709             // External tunnel table if needed or adding the ElanInterface to the
 710             // DpnInterfaces in the operational DS.
 711             holder.dpId = interfaceInfo.getDpId();
 712             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 713                 // FIXME: use elanInstaince.key() instead?
 714                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 715                 lock.lock();
 716                 try {
 717                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 718                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 719                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 720                     if (ElanUtils.isVlan(elanInstance)) {
 721                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 722                             elanInstanceName, existingElanDpnInterfaces);
 723                     } else {
 724                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 725                     }
 726                     if (holder.isFirstInterfaceInDpn) {
 727                         // ELAN's 1st ElanInterface added to this DPN
 728                         if (!existingElanDpnInterfaces.isPresent()) {
 729                             holder.dpnInterfaces =
 730                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 731                         } else {
 732                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 733                             List<String> elanInterfaces =
 734                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 735                             elanInterfaces.add(interfaceName);
 736                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 737                                 elanInterfaces, operTx);
 738                         }
 739                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 740                         // table, but only if Elan has VNI
 741                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 742                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 743                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 744                         }
 745                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 746                             interfaceName);
 747                     } else {
 748                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 749                         List<String> elanInterfaces =
 750                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 751                         elanInterfaces.add(interfaceName);
 752                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 753                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 754                                 interfaceName);
 755                         }
 756                         holder.dpnInterfaces =
 757                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 758                     }
 759                 } finally {
 760                     lock.unlock();
 761                 }
 762             }
 763
 764             // add code to install Local/Remote BC group, unknow DMAC entry,
 765             // terminating service table flow entry
 766             // call bindservice of interfacemanager to create ingress table flow
 767             // enty.
 768             // Add interface to the ElanInterfaceForwardingEntires Container
 769             createElanInterfaceTablesList(interfaceName, operTx);
 770         }));
 771         futures.forEach(ElanUtils::waitForTransactionToComplete);
 772         futures.add(
 773             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 774                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 775                     holder.isFirstInterfaceInDpn, confTx))));
 776
 777         // add the vlan provider interface to remote BC group for the elan
 778         // for internal vlan networks
 779         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 780             if (interfaceManager.isExternalInterface(interfaceName)) {
 781                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 782                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 783             }
 784         }
 785
 786         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 787             //update the remote-DPNs remoteBC group entry with Tunnels
 788             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 789                 holder.dpId);
 790             futures.add(
 791                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 792                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 793         }
 794
 795         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 796         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 797                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 798         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 799         return futures;
 800     }
 801
 802     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 803     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 804             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 805         String elanInstanceName = elanInstance.getElanInstanceName();
 806         String interfaceName = elanInterface.getName();
 807         List<ListenableFuture<Void>> futures = new ArrayList<>();
 808         BigInteger dpId = interfaceInfo.getDpId();
 809         boolean isInterfaceOperational = isOperational(interfaceInfo);
 810         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 811             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 812                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 813                     isFirstInterfaceInDpn, confTx);
 814
 815                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 816                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 817
 818                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 819                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 820                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 821                             staticMacEntry.getMacAddress());
 822                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 823                             LogicalDatastoreType.OPERATIONAL, macId);
 824                         if (existingMacEntry.isPresent()) {
 825                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 826                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 827                                 existingMacEntry.get(), operTx);
 828                         } else {
 829                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 830                                 interfaceName, staticMacEntry, operTx);
 831                         }
 832
 833                         if (isInterfaceOperational) {
 834                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 835                             // DPNs
 836                             String macAddress = staticMacEntry.getMacAddress().getValue();
 837                             LOG.info(
 838                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 839                                     + " {}",
 840                                 macAddress, elanInstanceName, interfaceName);
 841                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 842                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 843                         }
 844                     }
 845
 846                     if (isInterfaceOperational) {
 847                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 848                         // on purpose.
 849                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 850                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 851                         }
 852                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 853                             staticMacAddresses);
 854                     }
 855                 }
 856             }));
 857         }));
 858         futures.forEach(ElanUtils::waitForTransactionToComplete);
 859         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 860             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 861             try {
 862                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 863                 if (neutronPort != null) {
 864                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 865                 }
 866             } catch (IllegalArgumentException ex) {
 867                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 868             }
 869         }
 870         return futures;
 871     }
 872
 873     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 874             PhysAddress physAddress) {
 875         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 876         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 877         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 878                 LogicalDatastoreType.OPERATIONAL, macId);
 879
 880         if (!existingMacEntry.isPresent()) {
 881             return;
 882         }
 883
 884         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 885                 .withKey(new MacEntryKey(physAddress)).build();
 886         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 887                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 888     }
 889
 890     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 891             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 892         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 893         if (!existingElanDpnInterfaces.isPresent()) {
 894             return true;
 895         }
 896         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 897             return false;
 898         }
 899         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 900         int dummyInterfaceCount =  0;
 901         List<String> interfaces = dpnInterfaces.getInterfaces();
 902         if (interfaces == null) {
 903             return true;
 904         }
 905         if (interfaces.contains(routerPortUuid)) {
 906             dummyInterfaceCount++;
 907         }
 908         if (interfaces.contains(elanInstanceName)) {
 909             dummyInterfaceCount++;
 910         }
 911         return interfaces.size() == dummyInterfaceCount;
 912     }
 913
 914     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 915             PhysAddress physAddress) {
 916         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 917                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 918     }
 919
 920     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 921             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 922         if (!isOperational(interfaceInfo)) {
 923             return;
 924         }
 925         BigInteger dpId = interfaceInfo.getDpId();
 926         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 927             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 928         }
 929         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 930         if (isFirstInterfaceInDpn) {
 931             // Terminating Service , UnknownDMAC Table.
 932             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 933             // for ELAN Instance is VxLAN
 934             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 935                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 936             }
 937             setupUnknownDMacTable(elanInstance, dpId, confTx);
 938             /*
 939              * Install remote DMAC flow. This is required since this DPN is
 940              * added later to the elan instance and remote DMACs of other
 941              * interfaces in this elan instance are not present in the current
 942              * dpn.
 943              */
 944             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 945                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 946                         elanInstance.getElanInstanceName());
 947                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 948             }
 949         }
 950         // bind the Elan service to the Interface
 951         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 952     }
 953
 954     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 955             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 956         if (!isOperational(interfaceInfo)) {
 957             return;
 958         }
 959         // LocalBroadcast Group creation with elan-Interfaces
 960         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 961         if (isFirstInterfaceInDpn) {
 962             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 963             BigInteger dpId = interfaceInfo.getDpId();
 964             // RemoteBroadcast Group creation
 965             try {
 966                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 967             } catch (InterruptedException e1) {
 968                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 969             }
 970             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 971             try {
 972                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 973             } catch (InterruptedException e1) {
 974                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 975             }
 976         }
 977     }
 978
 979     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 980             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 981         int ifTag = interfaceInfo.getInterfaceTag();
 982         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 983                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 984                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 985                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 986                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 987
 988         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 989
 990         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 991                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 992                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 993                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 994
 995         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 996     }
 997
 998     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 999             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1000         int ifTag = interfaceInfo.getInterfaceTag();
1001         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1002             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1003
1004         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1005
1006         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1007             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1008             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1009             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1010
1011         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1012     }
1013
1014     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1015             TypedWriteTransaction<Configuration> confTx) {
1016         int elanTag = elanInfo.getElanTag().intValue();
1017         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1018         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1019         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1020         if (etreeInstance != null) {
1021             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1022             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1023             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1024         }
1025     }
1026
1027     @SuppressWarnings("checkstyle:IllegalCatch")
1028     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1029             TypedWriteTransaction<Configuration> confTx) {
1030         int bucketId = 0;
1031         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1032         if (elanDpns != null) {
1033             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1034             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1035                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1036                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1037                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1038                     List<Action> listAction = new ArrayList<>();
1039                     int actionKey = 0;
1040                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1041                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1042                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1043                     bucketId++;
1044                     for (DpnInterfaces otherFes : dpnInterfaces) {
1045                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1046                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1047                             && !otherFes.getInterfaces().isEmpty()) {
1048                             try {
1049                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1050                                         dpnInterface.getDpId(), otherFes.getDpId(),
1051                                         elanUtils.isOpenstackVniSemanticsEnforced()
1052                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1053                                 if (!remoteListActionInfo.isEmpty()) {
1054                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1055                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1056                                     bucketId++;
1057                                 }
1058                             } catch (Exception ex) {
1059                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1060                                         + "Logical Group Interface not found between source Dpn - {}, "
1061                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1062                                 return;
1063                             }
1064                         }
1065                     }
1066                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1067                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1068                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1069
1070                     if (remoteListBucketInfo.isEmpty()) {
1071                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1072                         continue;
1073                     }
1074                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1075                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1076                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1077                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1078                 }
1079             }
1080             try {
1081                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1082             } catch (InterruptedException e1) {
1083                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1084             }
1085         }
1086     }
1087
1088     private static List<MatchInfo> buildMatchesForVni(Long vni) {
1089         List<MatchInfo> mkMatches = new ArrayList<>();
1090         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1091         mkMatches.add(match);
1092         return mkMatches;
1093     }
1094
1095     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1096         List<InstructionInfo> mkInstructions = new ArrayList<>();
1097         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1098         return mkInstructions;
1099     }
1100
1101     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1102         List<MatchInfo> mkMatches = new ArrayList<>();
1103         // Matching metadata
1104         mkMatches.add(new MatchMetadata(
1105                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1106         return mkMatches;
1107     }
1108
1109     /**
1110      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1111      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1112      *
1113      * @param elanTag
1114      *            elanTag to be written in metadata when flow is selected
1115      * @return the instructions ready to be installed in a flow
1116      */
1117     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1118         List<InstructionInfo> mkInstructions = new ArrayList<>();
1119         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1120                 .getElanMetadataMask()));
1121         /* applicable for EXTERNAL_TUNNEL_TABLE only
1122         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1123         */
1124         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1125         return mkInstructions;
1126     }
1127
1128     // Install DMAC entry on dst DPN
1129     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1130             BigInteger dstDpId) {
1131         String interfaceName = interfaceInfo.getInterfaceName();
1132         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1133         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1134             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1135             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1136                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1137                     for (MacEntry macEntry : macEntries) {
1138                         String macAddress = macEntry.getMacAddress().getValue();
1139                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1140                             interfaceName);
1141                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1142                             interfaceInfo.getDpId())) {
1143                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1144                                 macAddress);
1145                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1146                         }
1147                     }
1148                 })));
1149         }
1150         return emptyList();
1151     }
1152
1153     private static void createDropBucket(List<Bucket> listBucket) {
1154         List<Action> actionsInfos = new ArrayList<>();
1155         actionsInfos.add(new ActionDrop().buildAction());
1156         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1157                 MDSALUtil.WATCH_GROUP);
1158         listBucket.add(dropBucket);
1159     }
1160
1161     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1162             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1163         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1164         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1165     }
1166
1167     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1168             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1169         List<Bucket> listBucket = new ArrayList<>();
1170         int bucketId = 0;
1171         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1172
1173         List<String> interfaces = new ArrayList<>();
1174         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1175             interfaces = newDpnInterface.getInterfaces();
1176         }
1177         for (String ifName : interfaces) {
1178             // In case if there is a InterfacePort in the cache which is not in
1179             // operational state, skip processing it
1180             InterfaceInfo ifInfo = interfaceManager
1181                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1182             if (!isOperational(ifInfo)) {
1183                 continue;
1184             }
1185
1186             if (!interfaceManager.isExternalInterface(ifName)) {
1187                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1188                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1189                 bucketId++;
1190             }
1191         }
1192
1193         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1194                 MDSALUtil.buildBucketLists(listBucket));
1195         LOG.trace("installing the localBroadCast Group:{}", group);
1196         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1197     }
1198
1199     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1200             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1201         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1202         if (etreeInstance != null) {
1203             List<Bucket> listBucket = new ArrayList<>();
1204             int bucketId = 0;
1205
1206             List<String> interfaces = new ArrayList<>();
1207             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1208                 interfaces = newDpnInterface.getInterfaces();
1209             }
1210             for (String ifName : interfaces) {
1211                 // In case if there is a InterfacePort in the cache which is not
1212                 // in
1213                 // operational state, skip processing it
1214                 InterfaceInfo ifInfo = interfaceManager
1215                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1216                 if (!isOperational(ifInfo)) {
1217                     continue;
1218                 }
1219
1220                 if (!interfaceManager.isExternalInterface(ifName)) {
1221                     // only add root interfaces
1222                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1223                 }
1224             }
1225
1226             if (listBucket.isEmpty()) { // No Buckets
1227                 createDropBucket(listBucket);
1228             }
1229
1230             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1231             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1232             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1233                     MDSALUtil.buildBucketLists(listBucket));
1234             LOG.trace("installing the localBroadCast Group:{}", group);
1235             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1236         }
1237     }
1238
1239     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1240             InterfaceInfo ifInfo) {
1241         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1242         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1243             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1244                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1245             bucketId++;
1246         }
1247         return bucketId;
1248     }
1249
1250     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1251             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1252             throws ExecutionException, InterruptedException {
1253         BigInteger dpnId = interfaceInfo.getDpId();
1254         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1255         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1256         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1257     }
1258
1259     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1260             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1261             throws ExecutionException, InterruptedException {
1262         BigInteger dpnId = interfaceInfo.getDpId();
1263         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1264         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1265         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1266     }
1267
1268     /**
1269      * Installs a flow in the External Tunnel table consisting in translating
1270      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1271      * elanTag that will be used later in the ELANs pipeline.
1272      * @param dpnId the dpn id
1273      */
1274     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1275             TypedWriteTransaction<Configuration> confTx) {
1276         long elanTag = elanInfo.getElanTag();
1277         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1278                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1279                 elanInfo.getElanInstanceName(), // flowName
1280                 0, // idleTimeout
1281                 0, // hardTimeout
1282                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1283                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1284                 getInstructionsIntOrExtTunnelTable(elanTag));
1285
1286         mdsalManager.addFlow(confTx, flowEntity);
1287     }
1288
1289     /**
1290      * Removes, from External Tunnel table, the flow that translates from VNI to
1291      * elanTag. Important: ensure this method is only called whenever there is
1292      * no other ElanInterface in the specified DPN
1293      *
1294      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1295      */
1296     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1297             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1298         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1299         // last ElanInstance plus
1300         // adding a new one does (almost at the same time) are executed in that
1301         // exact order
1302
1303         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1304         FlowEntity flowEntity = new FlowEntityBuilder()
1305             .setDpnId(dpnId)
1306             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1307             .setFlowId(flowId)
1308             .build();
1309         mdsalManager.removeFlow(confTx, flowEntity);
1310     }
1311
1312     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1313             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1314         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1315         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1316     }
1317
1318     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1319             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1320         List<? extends MatchInfoBase> listMatchInfoBase;
1321         List<InstructionInfo> instructionInfos;
1322         long serviceId;
1323         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1324             serviceId = elanTag;
1325             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1326             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1327         } else {
1328             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1329             listMatchInfoBase = buildMatchesForVni(serviceId);
1330             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1331         }
1332         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1333                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1334                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1335                 instructionInfos);
1336         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1337     }
1338
1339     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1340             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1341         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1342         if (etreeInstance != null) {
1343             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1344         }
1345     }
1346
1347     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1348             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1349         long elanTag = elanInfo.getElanTag();
1350         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1351         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1352         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1353     }
1354
1355     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1356             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1357         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1358         if (etreeLeafTag != null) {
1359             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1360             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1361             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1362         }
1363     }
1364
1365     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1366             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1367         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1368                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1369                 5, elanInfo.getElanInstanceName(), 0, 0,
1370                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1371                 getMatchesForElanTag(elanTag, /* SH flag */false),
1372                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1373
1374         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1375     }
1376
1377     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1378             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1379         // only if ELAN can connect to external network, perform the following
1380
1381         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1382             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1383                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1384                     5, elanInfo.getElanInstanceName(), 0, 0,
1385                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1386                     getMatchesForElanTag(elanTag, /* SH flag */true),
1387                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1388             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1389         }
1390     }
1391
1392
1393     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1394             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1395             throws ExecutionException, InterruptedException {
1396         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1397                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1398         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1399
1400         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1401             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1402                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1403                     .build();
1404             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1405         }
1406     }
1407
1408     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1409         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1410     }
1411
1412     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1413             TypedWriteTransaction<Configuration> tx) {
1414         if (isStandardElanService(elanInterface)) {
1415             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1416                     elanInterface.getName(), lportTag, tx);
1417         } else { // Etree service
1418             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1419         }
1420     }
1421
1422     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1423             TypedWriteTransaction<Configuration> tx) {
1424         int instructionKey = 0;
1425         List<Instruction> instructions = new ArrayList<>();
1426         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1427                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1428
1429         List<Action> actions = new ArrayList<>();
1430         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1431                 lportTag).buildAction());
1432         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1433                 elanTag).buildAction());
1434         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1435
1436         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1437                 ++instructionKey));
1438
1439         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1440         BoundServices serviceInfo = ElanUtils.getBoundServices(
1441                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1442                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1443         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1444         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1445                 bindServiceId);
1446         if (!existingElanService.isPresent()) {
1447             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1448         }
1449     }
1450
1451     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1452             TypedWriteTransaction<Configuration> tx) {
1453         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1454             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1455                     lportTag, tx);
1456         } else {
1457             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1458             if (etreeInstance == null) {
1459                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1460                         elanInterface.getName(), elanInfo.getElanInstanceName());
1461             } else {
1462                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1463                         elanInterface.getName(), lportTag, tx);
1464             }
1465         }
1466     }
1467
1468     private static boolean isStandardElanService(ElanInterface elanInterface) {
1469         return elanInterface.augmentation(EtreeInterface.class) == null;
1470     }
1471
1472     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1473             throws ExecutionException, InterruptedException {
1474         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1475         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1476         if (tx.read(bindServiceId).get().isPresent()) {
1477             tx.delete(bindServiceId);
1478         }
1479     }
1480
1481     private static String getFlowRef(long tableId, long elanTag) {
1482         return String.valueOf(tableId) + elanTag;
1483     }
1484
1485     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1486         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1487                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1488     }
1489
1490     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1491         return String.valueOf(tableId) + elanTag + shFlag;
1492     }
1493
1494     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1495         List<Action> listAction = new ArrayList<>();
1496         int actionKey = 0;
1497         listAction.add(
1498             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1499         actionKey++;
1500         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1501         return listAction;
1502     }
1503
1504     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1505             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1506         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1507                 .withKey(new DpnInterfacesKey(dpId)).build();
1508         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1509                 CREATE_MISSING_PARENTS);
1510         return dpnInterface;
1511     }
1512
1513     /**
1514      * Delete elan dpn interface from operational DS.
1515      *
1516      * @param elanInstanceName
1517      *            the elan instance name
1518      * @param dpId
1519      *            the dp id
1520      */
1521     private static void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1522             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1523         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1524                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1525         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1526         if (dpnInterfaces.isPresent()) {
1527             tx.delete(dpnInterfacesId);
1528         }
1529     }
1530
1531     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1532             BigInteger dpId, TypedWriteTransaction<Operational> tx) {
1533         List<String> interfaceNames = new ArrayList<>();
1534         interfaceNames.add(interfaceName);
1535         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1536                 .withKey(new DpnInterfacesKey(dpId)).build();
1537         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1538                 CREATE_MISSING_PARENTS);
1539         return dpnInterface;
1540     }
1541
1542     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1543             throws ExecutionException, InterruptedException {
1544         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1545                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1546         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1547         // Adding new Elan Interface Port to the operational DataStore without
1548         // Static-Mac Entries..
1549         if (!interfaceMacTables.isPresent()) {
1550             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1551                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1552             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1553                     CREATE_MISSING_PARENTS);
1554         }
1555     }
1556
1557     private static void createElanStateList(String elanInstanceName, String interfaceName,
1558             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1559         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1560         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1561         // Adding new Elan Interface Port to the operational DataStore without
1562         // Static-Mac Entries..
1563         if (elanInterfaceLists.isPresent()) {
1564             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1565             if (interfaceLists == null) {
1566                 interfaceLists = new ArrayList<>();
1567             }
1568             interfaceLists.add(interfaceName);
1569             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1570                     .withKey(new ElanKey(elanInstanceName)).build();
1571             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1572         }
1573     }
1574
1575     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1576         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1577     }
1578
1579     @SuppressWarnings("checkstyle:IllegalCatch")
1580     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1581         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1582         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1583                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1584         if (dpnInterfaceLists == null) {
1585             return;
1586         }
1587         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1588         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1589             int cnt = 0;
1590             String elanName = elanDpns.getElanInstanceName();
1591             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1592             if (elanInfo == null) {
1593                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1594                         + "skipping this ELAN for tunnel handling", elanName);
1595                 continue;
1596             }
1597             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1598                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1599                 continue;
1600             }
1601             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1602             if (dpnInterfaces == null) {
1603                 continue;
1604             }
1605             DpnInterfaces dstDpnIf = null;
1606             for (DpnInterfaces dpnIf : dpnInterfaces) {
1607                 BigInteger dpnIfDpId = dpnIf.getDpId();
1608                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1609                     cnt++;
1610                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1611                     cnt++;
1612                     dstDpnIf = dpnIf;
1613                 }
1614             }
1615             if (cnt == 2) {
1616                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1617                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1618                 jobCoordinator.enqueueJob(elanName, () -> {
1619                     // update Remote BC Group
1620                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1621                     try {
1622                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1623                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1624                                 confTx)).get();
1625                     } catch (RuntimeException e) {
1626                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1627                     }
1628                     Set<String> interfaceLists = new HashSet<>();
1629                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1630                     for (String ifName : interfaceLists) {
1631                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1632                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1633                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1634                             if (isOperational(interfaceInfo)) {
1635                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1636                             }
1637                             return emptyList();
1638                         }, ElanConstants.JOB_MAX_RETRIES);
1639                     }
1640                     return emptyList();
1641                 }, ElanConstants.JOB_MAX_RETRIES);
1642             }
1643
1644         }
1645     }
1646
1647     /**
1648      * Handle external tunnel state event.
1649      *
1650      * @param externalTunnel
1651      *            the external tunnel
1652      * @param intrf
1653      *            the interface
1654      */
1655     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1656         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1657             return;
1658         }
1659         // dpId/externalNodeId will be available either in source or destination
1660         // based on the tunnel end point
1661         BigInteger dpId = null;
1662         NodeId externalNodeId = null;
1663         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1664             dpId = new BigInteger(externalTunnel.getSourceDevice());
1665             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1666         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1667             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1668             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1669         }
1670         if (dpId == null || externalNodeId == null) {
1671             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1672             return;
1673         }
1674
1675         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1676         if (dpnInterfaceLists == null) {
1677             return;
1678         }
1679         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1680         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1681             String elanName = elanDpns.getElanInstanceName();
1682             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1683
1684             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1685             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1686                     || dpnInterfaces.getInterfaces().isEmpty()) {
1687                 continue;
1688             }
1689             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1690
1691             final BigInteger finalDpId = dpId;
1692             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1693                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1694                 "Error setting up ELAN BGs");
1695             // install L2gwDevices local macs in dpn.
1696             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1697             // Install dpn macs on external device
1698             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1699                     externalNodeId);
1700         }
1701         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1702     }
1703
1704     /**
1705      * Installs dpn macs in external device. first it checks if the physical
1706      * locator towards this dpn tep is present or not if the physical locator is
1707      * present go ahead and add the ucast macs otherwise update the mcast mac
1708      * entry to include this dpn tep ip and schedule the job to put ucast macs
1709      * once the physical locator is programmed in device
1710      *
1711      * @param elanName
1712      *            the elan name
1713      * @param lstElanInterfaceNames
1714      *            the lst Elan interface names
1715      * @param dpnId
1716      *            the dpn id
1717      * @param externalNodeId
1718      *            the external node id
1719      */
1720     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1721             NodeId externalNodeId) {
1722         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1723                 externalNodeId.getValue());
1724         if (elanL2GwDevice == null) {
1725             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1726             return;
1727         }
1728         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1729         if (dpnTepIp == null) {
1730             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1731                     externalNodeId);
1732             return;
1733         }
1734
1735         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1736         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1737                 LogicalDatastoreType.OPERATIONAL);
1738         boolean phyLocAlreadyExists =
1739                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1740                 dpnTepIp);
1741         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1742                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1743         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1744
1745         if (phyLocAlreadyExists) {
1746             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1747             return;
1748         }
1749         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1750         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1751     }
1752
1753     /**
1754      * Validate external tunnel state event.
1755      *
1756      * @param externalTunnel
1757      *            the external tunnel
1758      * @param intrf
1759      *            the intrf
1760      * @return true, if successful
1761      */
1762     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1763         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1764             String srcDevice = externalTunnel.getDestinationDevice();
1765             String destDevice = externalTunnel.getSourceDevice();
1766             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1767                     LogicalDatastoreType.CONFIGURATION);
1768             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1769                     otherEndPointExtTunnel);
1770             if (otherEndPointExtTunnel != null) {
1771                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1772                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1773                 if (otherEndPointInterfaceOperational) {
1774                     return true;
1775                 } else {
1776                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1777                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1778                 }
1779             }
1780         }
1781         return false;
1782     }
1783
1784     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1785         List<MatchInfo> mkMatches = new ArrayList<>();
1786         // Matching metadata
1787         mkMatches.add(
1788                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1789         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1790         return mkMatches;
1791     }
1792
1793     @Override
1794     protected ElanInterfaceManager getDataTreeChangeListener() {
1795         return this;
1796     }
1797
1798     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1799                                              BigInteger dpId) {
1800         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1801         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1802             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1803                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1804         try {
1805             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1806         } catch (InterruptedException e) {
1807             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1808         }
1809     }
1810 }