elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.requireNonNullElse;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 135 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 /**
 141  * Class in charge of handling creations, modifications and removals of
 142  * ElanInterfaces.
 143  *
 144  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 145  */
 146 @Singleton
 147 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 148         implements RecoverableListener {
 149     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final boolean SH_FLAG_SET = true;
 152     private static final boolean SH_FLAG_UNSET = false;
 153
 154     private final DataBroker broker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalManager;
 157     private final IInterfaceManager interfaceManager;
 158     private final IdManagerService idManager;
 159     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 160     private final INeutronVpnManager neutronVpnManager;
 161     private final ElanItmUtils elanItmUtils;
 162     private final ElanEtreeUtils elanEtreeUtils;
 163     private final ElanL2GatewayUtils elanL2GatewayUtils;
 164     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 165     private final ElanUtils elanUtils;
 166     private final JobCoordinator jobCoordinator;
 167     private final ElanInstanceCache elanInstanceCache;
 168     private final ElanInterfaceCache elanInterfaceCache;
 169
 170     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 171         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 172
 173     @Inject
 174     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 175                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 176                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 177                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 178                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 179                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 180                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 181                                 final ElanInstanceCache elanInstanceCache,
 182                                 final ElanInterfaceCache elanInterfaceCache,
 183                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 184                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 185         super(ElanInterface.class, ElanInterfaceManager.class);
 186         this.broker = dataBroker;
 187         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 188         this.idManager = managerService;
 189         this.mdsalManager = mdsalApiManager;
 190         this.interfaceManager = interfaceManager;
 191         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 192         this.neutronVpnManager = neutronVpnManager;
 193         this.elanItmUtils = elanItmUtils;
 194         this.elanEtreeUtils = elanEtreeUtils;
 195         this.elanL2GatewayUtils = elanL2GatewayUtils;
 196         this.elanUtils = elanUtils;
 197         this.jobCoordinator = jobCoordinator;
 198         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 199         this.elanInstanceCache = elanInstanceCache;
 200         this.elanInterfaceCache = elanInterfaceCache;
 201         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 202     }
 203
 204     @Override
 205     @PostConstruct
 206     public void init() {
 207         registerListener();
 208     }
 209
 210     @Override
 211     public void registerListener() {
 212         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 213     }
 214
 215     @Override
 216     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 217         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 218     }
 219
 220     @Override
 221     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 222         String interfaceName = del.getName();
 223         String elanInstanceName = del.getElanInstanceName();
 224         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 225         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 226             elanInterfaces.remove(del);
 227             if (elanInterfaces.isEmpty()) {
 228                 unProcessedElanInterfaces.remove(elanInstanceName);
 229             }
 230         }
 231         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 232         /*
 233          * Handling in case the elan instance is deleted.If the Elan instance is
 234          * deleted, there is no need to explicitly delete the elan interfaces
 235          */
 236         if (elanInfo == null) {
 237             return;
 238         }
 239         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 240         if (interfaceInfo == null && elanInfo.isExternal()) {
 241             // In deleting external network, the underlying ietf Inteface might have been removed
 242             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 243             // from Operational DS instead
 244             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 245         }
 246         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 247                 interfaceName, interfaceInfo, this);
 248         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 249     }
 250
 251     private static class RemoveElanInterfaceHolder {
 252         boolean isLastElanInterface = false;
 253         boolean isLastInterfaceOnDpn = false;
 254         BigInteger dpId = null;
 255     }
 256
 257     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 258     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 259             InterfaceInfo interfaceInfo) {
 260         String elanName = elanInfo.getElanInstanceName();
 261         long elanTag = elanInfo.getElanTag();
 262         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 263         List<ListenableFuture<Void>> futures = new ArrayList<>();
 264         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 265         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 266             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 267             if (elanState == null) {
 268                 return;
 269             }
 270             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 271                 List<String> elanInterfaces = requireNonNullElse(elanState.getElanInterfaces(), emptyList());
 272                 if (elanInterfaces.isEmpty()) {
 273                     holder.isLastElanInterface = true;
 274                 }
 275                 if (interfaceInfo != null) {
 276                     holder.dpId = interfaceInfo.getDpId();
 277                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 278                         interfaceName, elanTag, interfaceTx);
 279                     /*
 280                      * If there are not elan ports, remove the unknown dmac, terminating
 281                      * service table flows, remote/local bc group
 282                      */
 283                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 284                         || dpnInterfaces.getInterfaces().isEmpty()) {
 285                         // No more Elan Interfaces in this DPN
 286                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 287                             holder.dpId);
 288                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 289                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 290                         }
 291                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 292                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 293                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 296                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 297                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 299                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 300                             }
 301                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 302                         }
 303                         holder.isLastInterfaceOnDpn = true;
 304                     } else {
 305                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 306                     }
 307                 }
 308             }));
 309         }));
 310         futures.forEach(ElanUtils::waitForTransactionToComplete);
 311
 312         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 313             futures.add(
 314                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 315                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 316         }
 317         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 318                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 319         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 320                 ElanConstants.JOB_MAX_RETRIES);
 321
 322         return futures;
 323     }
 324
 325     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 326             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 327             throws ExecutionException, InterruptedException {
 328         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 329         if (etreeLeafTag != null) {
 330             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 331             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 332         }
 333     }
 334
 335     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 336             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 337             throws ExecutionException, InterruptedException {
 338         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340     }
 341
 342     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 349             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 350             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 351         }
 352     }
 353
 354     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 358         if (etreeInstance != null) {
 359             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 360             BigInteger dpnId = interfaceInfo.getDpId();
 361             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 362             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 363             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 364         }
 365     }
 366
 367     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 368             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 369         String elanName = elanInfo.getElanInstanceName();
 370         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 371         if (elanState == null) {
 372             return elanState;
 373         }
 374         List<String> elanInterfaces = requireNonNullElse(elanState.getElanInterfaces(), emptyList());
 375         boolean isRemoved = elanInterfaces.remove(interfaceName);
 376         if (!isRemoved) {
 377             return elanState;
 378         }
 379
 380         if (elanInterfaces.isEmpty()) {
 381             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 384         } else {
 385             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 386                     .withKey(new ElanKey(elanName)).build();
 387             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 388         }
 389         return elanState;
 390     }
 391
 392     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 393             throws ReadFailedException {
 394         // removing the ElanInterface from the config data_store if interface is
 395         // not present in Interface config DS
 396         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 397             interfaceName) == null
 398                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 399             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 400         }
 401     }
 402
 403     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 404             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 405         String elanName = elanInfo.getElanInstanceName();
 406         List<ListenableFuture<Void>> futures = new ArrayList<>();
 407         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 408             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 409                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 410                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 411                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 412                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 413                 if (interfaceInfo != null) {
 414                     if (existingElanInterfaceMac.isPresent()) {
 415                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 416                         if (existingMacEntries != null) {
 417                             List<PhysAddress> macAddresses = new ArrayList<>();
 418                             for (MacEntry macEntry : existingMacEntries) {
 419                                 PhysAddress macAddress = macEntry.getMacAddress();
 420                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 421                                         macAddress.getValue(), interfaceName);
 422                                 Optional<MacEntry> macEntryOptional =
 423                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 424                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 425                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 426                                 }
 427                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 428                                 macAddresses.add(macAddress);
 429                             }
 430
 431                             // Removing all those MACs from External Devices belonging
 432                             // to this ELAN
 433                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 434                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 435                             }
 436                         }
 437                     }
 438                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 439                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 440                 } else if (existingElanInterfaceMac.isPresent()) {
 441                     // Interface does not exist in ConfigDS, so lets remove everything
 442                     // about that interface related to Elan
 443                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 444                     if (macEntries != null) {
 445                         for (MacEntry macEntry : macEntries) {
 446                             PhysAddress macAddress = macEntry.getMacAddress();
 447                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 448                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 449                             }
 450                         }
 451                     }
 452                 }
 453                 if (existingElanInterfaceMac.isPresent()) {
 454                     interfaceTx.delete(elanInterfaceId);
 455                 }
 456                 unbindService(interfaceName, flowTx);
 457                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 458             }));
 459         }));
 460         return futures;
 461     }
 462
 463     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 464                                                                          String interfaceName, long elanTag,
 465                                                                          TypedReadWriteTransaction<Operational> tx)
 466             throws ExecutionException, InterruptedException {
 467         // FIXME: pass in and use ElanInstanceKey instead?
 468         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 469         lock.lock();
 470         try {
 471             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 472             if (dpnInterfaces != null) {
 473                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 474                 if (interfaceLists != null) {
 475                     interfaceLists.remove(interfaceName);
 476                 }
 477
 478                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 479                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 480                     deleteElanDpnInterface(elanName, dpId, tx);
 481                 } else {
 482                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 483                 }
 484             }
 485             return dpnInterfaces;
 486         } finally {
 487             lock.unlock();
 488         }
 489     }
 490
 491     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 492         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 493         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 494             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 495                 BigInteger currentDpId = dpnInterface.getDpId();
 496                 if (!currentDpId.equals(dpId)) {
 497                     for (String elanInterface : requireNonNullElse(dpnInterface.getInterfaces(),
 498                             Collections.<String>emptyList())) {
 499                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 500                         if (macs == null || macs.getMacEntry() == null) {
 501                             continue;
 502                         }
 503                         for (MacEntry mac : macs.getMacEntry()) {
 504                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 505                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 506                         }
 507                     }
 508                 }
 509             }
 510         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 511     }
 512
 513     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 514             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 515         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 516         if (etreeLeafTag != null) {
 517             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 518         }
 519     }
 520
 521     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 522             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 523         mdsalManager
 524                 .removeFlow(confTx, dpId,
 525                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 526                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 527                                         mac.getMacAddress().getValue(), elanTag)));
 528     }
 529
 530     /*
 531     * Possible Scenarios for update
 532     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 533         then all {1,2,3,4} should be removed
 534
 535         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 536         then all {1,2,3,4} should be added
 537
 538         c. if orig = {1,2,3,4} updated={2,3,4}
 539         then 1 should be removed
 540
 541         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 542         then we should just add 5
 543
 544         e. if orig = {1,2,3,4} updated={2,3,4,5}
 545         then 1 should be removed , 5 should be added
 546     * */
 547     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 548     @Override
 549     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 550         // updating the static-Mac Entries for the existing elanInterface
 551         String elanName = update.getElanInstanceName();
 552         String interfaceName = update.getName();
 553
 554         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 555         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 556         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 557         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 558
 559         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 560                 deletedEntry.getMacAddress()));
 561
 562         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 563         * Hence add the macentries for the same.*/
 564         for (StaticMacEntries staticMacEntry : updatedEntries) {
 565             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 566                     staticMacEntry.getMacAddress());
 567             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 568                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 569                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 570                     if (existingMacEntry.isPresent()) {
 571                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 572                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 573                             tx);
 574                     } else {
 575                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 576                             elanName, interfaceName, staticMacEntry, tx);
 577                     }
 578                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 579         }
 580     }
 581
 582     @Override
 583     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 584         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 585             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 586             String interfaceName = elanInterfaceAdded.getName();
 587             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 588             if (interfaceInfo == null) {
 589                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 590                 return;
 591             }
 592             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 593
 594             if (elanInstance == null) {
 595                 // Add the ElanInstance in the Configuration data-store
 596                 List<String> elanInterfaces = new ArrayList<>();
 597                 elanInterfaces.add(interfaceName);
 598                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 599                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 600                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 601                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 602             }
 603
 604             Long elanTag = elanInstance.getElanTag();
 605             // If elan tag is not updated, then put the elan interface into
 606             // unprocessed entry map and entry. Let entries
 607             // in this map get processed during ELAN update DCN.
 608             if (elanTag == null || elanTag == 0L) {
 609                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 610                 if (elanInterfaces == null) {
 611                     elanInterfaces = new ConcurrentLinkedQueue<>();
 612                 }
 613                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 614                     elanInterfaces.add(elanInterfaceAdded);
 615                 }
 616                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 617                 return;
 618             }
 619             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 620                     interfaceInfo, elanInstance, this);
 621             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 622         }), LOG, "Error processing added ELAN interface");
 623     }
 624
 625     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 626         List<ListenableFuture<Void>> futures = new ArrayList<>();
 627         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 628         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 629             return futures;
 630         }
 631         for (ElanInterface elanInterface : elanInterfaces) {
 632             String interfaceName = elanInterface.getName();
 633             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 634             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 635         }
 636         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 637         return futures;
 638     }
 639
 640     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 641             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 642         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 643                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 644         List<DpnInterfaces> dpnInterfaceLists = null;
 645         if (elanDpnInterfacesList != null) {
 646             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 647         }
 648         if (dpnInterfaceLists == null) {
 649             dpnInterfaceLists = new ArrayList<>();
 650         }
 651         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 652             BigInteger dstDpId = interfaceInfo.getDpId();
 653             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 654                 continue;
 655             }
 656             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 657             for (String remoteIf : remoteElanInterfaces) {
 658                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 659                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 660                 if (elanIfMac == null || remoteInterface == null) {
 661                     continue;
 662                 }
 663                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 664                 if (remoteMacEntries != null) {
 665                     for (MacEntry macEntry : remoteMacEntries) {
 666                         String macAddress = macEntry.getMacAddress().getValue();
 667                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 668                                 dstDpId, elanInstance.getElanInstanceName());
 669                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 670                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 671                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 672                     }
 673                 }
 674             }
 675         }
 676     }
 677
 678     private static class AddElanInterfaceHolder {
 679         private DpnInterfaces dpnInterfaces = null;
 680         private boolean isFirstInterfaceInDpn = false;
 681         private BigInteger dpId;
 682     }
 683
 684     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 685     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 686             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 687         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 688         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 689         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 690
 691         String interfaceName = elanInterface.getName();
 692         String elanInstanceName = elanInterface.getElanInstanceName();
 693
 694         List<ListenableFuture<Void>> futures = new ArrayList<>();
 695         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 696         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 697             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 698             if (elanInfo == null) {
 699                 List<String> elanInterfaces = new ArrayList<>();
 700                 elanInterfaces.add(interfaceName);
 701                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 702                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 703                         operTx)));
 704             } else {
 705                 createElanStateList(elanInstanceName, interfaceName, operTx);
 706             }
 707             // Specific actions to the DPN where the ElanInterface has been added,
 708             // for example, programming the
 709             // External tunnel table if needed or adding the ElanInterface to the
 710             // DpnInterfaces in the operational DS.
 711             holder.dpId = interfaceInfo.getDpId();
 712             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 713                 // FIXME: use elanInstaince.key() instead?
 714                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 715                 lock.lock();
 716                 try {
 717                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 718                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 719                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 720                     if (ElanUtils.isVlan(elanInstance)) {
 721                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 722                             elanInstanceName, existingElanDpnInterfaces);
 723                     } else {
 724                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 725                     }
 726                     if (holder.isFirstInterfaceInDpn) {
 727                         // ELAN's 1st ElanInterface added to this DPN
 728                         if (!existingElanDpnInterfaces.isPresent()) {
 729                             holder.dpnInterfaces =
 730                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 731                         } else {
 732                             List<String> elanInterfaces =
 733                                 requireNonNullElse(existingElanDpnInterfaces.get().getInterfaces(), emptyList());
 734                             elanInterfaces.add(interfaceName);
 735                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 736                                 elanInterfaces, operTx);
 737                         }
 738                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 739                         // table, but only if Elan has VNI
 740                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 741                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 742                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 743                         }
 744                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 745                             interfaceName);
 746                     } else {
 747                         List<String> elanInterfaces =
 748                             requireNonNullElse(existingElanDpnInterfaces.get().getInterfaces(), emptyList());
 749                         elanInterfaces.add(interfaceName);
 750                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 751                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 752                                 interfaceName);
 753                         }
 754                         holder.dpnInterfaces =
 755                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 756                     }
 757                 } finally {
 758                     lock.unlock();
 759                 }
 760             }
 761
 762             // add code to install Local/Remote BC group, unknow DMAC entry,
 763             // terminating service table flow entry
 764             // call bindservice of interfacemanager to create ingress table flow
 765             // enty.
 766             // Add interface to the ElanInterfaceForwardingEntires Container
 767             createElanInterfaceTablesList(interfaceName, operTx);
 768         }));
 769         futures.forEach(ElanUtils::waitForTransactionToComplete);
 770         futures.add(
 771             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 772                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 773                     holder.isFirstInterfaceInDpn, confTx))));
 774
 775         // add the vlan provider interface to remote BC group for the elan
 776         // for internal vlan networks
 777         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 778             if (interfaceManager.isExternalInterface(interfaceName)) {
 779                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 780                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 781             }
 782         }
 783
 784         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 785             //update the remote-DPNs remoteBC group entry with Tunnels
 786             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 787                 holder.dpId);
 788             futures.add(
 789                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 790                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 791         }
 792
 793         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 794         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 795                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 796         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 797         return futures;
 798     }
 799
 800     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 801     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 802             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 803         String elanInstanceName = elanInstance.getElanInstanceName();
 804         String interfaceName = elanInterface.getName();
 805         List<ListenableFuture<Void>> futures = new ArrayList<>();
 806         BigInteger dpId = interfaceInfo.getDpId();
 807         boolean isInterfaceOperational = isOperational(interfaceInfo);
 808         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 809             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 810                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 811                     isFirstInterfaceInDpn, confTx, operTx);
 812
 813                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 814                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 815
 816                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 817                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 818                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 819                             staticMacEntry.getMacAddress());
 820                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 821                             LogicalDatastoreType.OPERATIONAL, macId);
 822                         if (existingMacEntry.isPresent()) {
 823                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 824                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 825                                 existingMacEntry.get(), operTx);
 826                         } else {
 827                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 828                                 interfaceName, staticMacEntry, operTx);
 829                         }
 830
 831                         if (isInterfaceOperational) {
 832                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 833                             // DPNs
 834                             String macAddress = staticMacEntry.getMacAddress().getValue();
 835                             LOG.info(
 836                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 837                                     + " {}",
 838                                 macAddress, elanInstanceName, interfaceName);
 839                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 840                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 841                         }
 842                     }
 843
 844                     if (isInterfaceOperational) {
 845                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 846                         // on purpose.
 847                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 848                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 849                         }
 850                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 851                             staticMacAddresses);
 852                     }
 853                 }
 854             }));
 855         }));
 856         futures.forEach(ElanUtils::waitForTransactionToComplete);
 857         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 858             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 859             try {
 860                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 861                 if (neutronPort != null) {
 862                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 863                 }
 864             } catch (IllegalArgumentException ex) {
 865                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 866             }
 867         }
 868         return futures;
 869     }
 870
 871     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 872             PhysAddress physAddress) {
 873         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 874         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 875         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 876                 LogicalDatastoreType.OPERATIONAL, macId);
 877
 878         if (!existingMacEntry.isPresent()) {
 879             return;
 880         }
 881
 882         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 883                 .withKey(new MacEntryKey(physAddress)).build();
 884         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 885                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 886     }
 887
 888     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 889             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 890         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 891         if (!existingElanDpnInterfaces.isPresent()) {
 892             return true;
 893         }
 894         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 895             return false;
 896         }
 897         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 898         int dummyInterfaceCount =  0;
 899         List<String> interfaces = requireNonNullElse(dpnInterfaces.getInterfaces(), emptyList());
 900         if (interfaces.contains(routerPortUuid)) {
 901             dummyInterfaceCount++;
 902         }
 903         if (interfaces.contains(elanInstanceName)) {
 904             dummyInterfaceCount++;
 905         }
 906         return interfaces.size() - dummyInterfaceCount == 0;
 907     }
 908
 909     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 910         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 911                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 912     }
 913
 914     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 915             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx,
 916             TypedWriteTransaction<Operational> operTx) {
 917         if (!isOperational(interfaceInfo)) {
 918             return;
 919         }
 920         BigInteger dpId = interfaceInfo.getDpId();
 921         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 922             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 923         }
 924         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 925         if (isFirstInterfaceInDpn) {
 926             // Terminating Service , UnknownDMAC Table.
 927             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 928             // for ELAN Instance is VxLAN
 929             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 930                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 931             }
 932             setupUnknownDMacTable(elanInstance, dpId, confTx);
 933             /*
 934              * Install remote DMAC flow. This is required since this DPN is
 935              * added later to the elan instance and remote DMACs of other
 936              * interfaces in this elan instance are not present in the current
 937              * dpn.
 938              */
 939             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 940                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 941                         elanInstance.getElanInstanceName());
 942                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 943             }
 944         }
 945         // bind the Elan service to the Interface
 946         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 947     }
 948
 949     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 950             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 951         if (!isOperational(interfaceInfo)) {
 952             return;
 953         }
 954         // LocalBroadcast Group creation with elan-Interfaces
 955         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 956         if (isFirstInterfaceInDpn) {
 957             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 958             BigInteger dpId = interfaceInfo.getDpId();
 959             // RemoteBroadcast Group creation
 960             try {
 961                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 962             } catch (InterruptedException e1) {
 963                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 964             }
 965             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 966             try {
 967                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 968             } catch (InterruptedException e1) {
 969                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 970             }
 971         }
 972     }
 973
 974     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 975             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 976         int ifTag = interfaceInfo.getInterfaceTag();
 977         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 978                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 979                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 980                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 981                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 982
 983         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 984
 985         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 986                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 987                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 988                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 989
 990         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 991     }
 992
 993     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 994             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 995         int ifTag = interfaceInfo.getInterfaceTag();
 996         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 997             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
 998
 999         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1000
1001         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1002             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1003             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1004             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1005
1006         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1007     }
1008
1009     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1010             TypedWriteTransaction<Configuration> confTx) {
1011         int elanTag = elanInfo.getElanTag().intValue();
1012         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1013         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1014         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1015         if (etreeInstance != null) {
1016             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1017             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1018             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1019         }
1020     }
1021
1022     @SuppressWarnings("checkstyle:IllegalCatch")
1023     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1024             TypedWriteTransaction<Configuration> confTx) {
1025         int bucketId = 0;
1026         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1027         if (elanDpns != null) {
1028             List<DpnInterfaces> dpnInterfaces = requireNonNullElse(elanDpns.getDpnInterfaces(), emptyList());
1029             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1030                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1031                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1032                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1033                     List<Action> listAction = new ArrayList<>();
1034                     int actionKey = 0;
1035                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1036                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1037                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1038                     bucketId++;
1039                     for (DpnInterfaces otherFes : dpnInterfaces) {
1040                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1041                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1042                             && !otherFes.getInterfaces().isEmpty()) {
1043                             try {
1044                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1045                                         dpnInterface.getDpId(), otherFes.getDpId(),
1046                                         elanUtils.isOpenstackVniSemanticsEnforced()
1047                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1048                                 if (!remoteListActionInfo.isEmpty()) {
1049                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1050                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1051                                     bucketId++;
1052                                 }
1053                             } catch (Exception ex) {
1054                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1055                                         + "Logical Group Interface not found between source Dpn - {}, "
1056                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1057                                 return;
1058                             }
1059                         }
1060                     }
1061                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1062                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1063                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1064
1065                     if (remoteListBucketInfo.isEmpty()) {
1066                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1067                         continue;
1068                     }
1069                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1070                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1071                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1072                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1073                 }
1074             }
1075             try {
1076                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1077             } catch (InterruptedException e1) {
1078                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1079             }
1080         }
1081     }
1082
1083     private List<MatchInfo> buildMatchesForVni(Long vni) {
1084         List<MatchInfo> mkMatches = new ArrayList<>();
1085         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1086         mkMatches.add(match);
1087         return mkMatches;
1088     }
1089
1090     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1091         List<InstructionInfo> mkInstructions = new ArrayList<>();
1092         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1093         return mkInstructions;
1094     }
1095
1096     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1097         List<MatchInfo> mkMatches = new ArrayList<>();
1098         // Matching metadata
1099         mkMatches.add(new MatchMetadata(
1100                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1101         return mkMatches;
1102     }
1103
1104     /**
1105      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1106      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1107      *
1108      * @param elanTag
1109      *            elanTag to be written in metadata when flow is selected
1110      * @return the instructions ready to be installed in a flow
1111      */
1112     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1113         List<InstructionInfo> mkInstructions = new ArrayList<>();
1114         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1115                 .getElanMetadataMask()));
1116         /* applicable for EXTERNAL_TUNNEL_TABLE only
1117         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1118         */
1119         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1120         return mkInstructions;
1121     }
1122
1123     // Install DMAC entry on dst DPN
1124     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1125             BigInteger dstDpId) {
1126         String interfaceName = interfaceInfo.getInterfaceName();
1127         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1128         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1129             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1130             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1131                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1132                     for (MacEntry macEntry : macEntries) {
1133                         String macAddress = macEntry.getMacAddress().getValue();
1134                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1135                             interfaceName);
1136                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1137                             interfaceInfo.getDpId())) {
1138                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1139                                 macAddress);
1140                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1141                         }
1142                     }
1143                 })));
1144         }
1145         return emptyList();
1146     }
1147
1148     private void createDropBucket(List<Bucket> listBucket) {
1149         List<Action> actionsInfos = new ArrayList<>();
1150         actionsInfos.add(new ActionDrop().buildAction());
1151         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1152                 MDSALUtil.WATCH_GROUP);
1153         listBucket.add(dropBucket);
1154     }
1155
1156     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1157             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1158         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1159         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1160     }
1161
1162     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1163             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1164         List<Bucket> listBucket = new ArrayList<>();
1165         int bucketId = 0;
1166         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1167
1168         List<String> interfaces = new ArrayList<>();
1169         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1170             interfaces = newDpnInterface.getInterfaces();
1171         }
1172         for (String ifName : interfaces) {
1173             // In case if there is a InterfacePort in the cache which is not in
1174             // operational state, skip processing it
1175             InterfaceInfo ifInfo = interfaceManager
1176                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1177             if (!isOperational(ifInfo)) {
1178                 continue;
1179             }
1180
1181             if (!interfaceManager.isExternalInterface(ifName)) {
1182                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1183                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1184                 bucketId++;
1185             }
1186         }
1187
1188         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1189                 MDSALUtil.buildBucketLists(listBucket));
1190         LOG.trace("installing the localBroadCast Group:{}", group);
1191         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1192     }
1193
1194     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1195             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1196         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1197         if (etreeInstance != null) {
1198             List<Bucket> listBucket = new ArrayList<>();
1199             int bucketId = 0;
1200
1201             List<String> interfaces = new ArrayList<>();
1202             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1203                 interfaces = newDpnInterface.getInterfaces();
1204             }
1205             for (String ifName : interfaces) {
1206                 // In case if there is a InterfacePort in the cache which is not
1207                 // in
1208                 // operational state, skip processing it
1209                 InterfaceInfo ifInfo = interfaceManager
1210                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1211                 if (!isOperational(ifInfo)) {
1212                     continue;
1213                 }
1214
1215                 if (!interfaceManager.isExternalInterface(ifName)) {
1216                     // only add root interfaces
1217                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1218                 }
1219             }
1220
1221             if (listBucket.isEmpty()) { // No Buckets
1222                 createDropBucket(listBucket);
1223             }
1224
1225             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1226             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1227             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1228                     MDSALUtil.buildBucketLists(listBucket));
1229             LOG.trace("installing the localBroadCast Group:{}", group);
1230             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1231         }
1232     }
1233
1234     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1235             InterfaceInfo ifInfo) {
1236         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1237         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1238             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1239                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1240             bucketId++;
1241         }
1242         return bucketId;
1243     }
1244
1245     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1246             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1247             throws ExecutionException, InterruptedException {
1248         BigInteger dpnId = interfaceInfo.getDpId();
1249         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1250         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1251         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1252     }
1253
1254     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1255             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1256             throws ExecutionException, InterruptedException {
1257         BigInteger dpnId = interfaceInfo.getDpId();
1258         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1259         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1260         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1261     }
1262
1263     /**
1264      * Installs a flow in the External Tunnel table consisting in translating
1265      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1266      * elanTag that will be used later in the ELANs pipeline.
1267      * @param dpnId the dpn id
1268      */
1269     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1270             TypedWriteTransaction<Configuration> confTx) {
1271         long elanTag = elanInfo.getElanTag();
1272         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1273                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1274                 elanInfo.getElanInstanceName(), // flowName
1275                 0, // idleTimeout
1276                 0, // hardTimeout
1277                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1278                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1279                 getInstructionsIntOrExtTunnelTable(elanTag));
1280
1281         mdsalManager.addFlow(confTx, flowEntity);
1282     }
1283
1284     /**
1285      * Removes, from External Tunnel table, the flow that translates from VNI to
1286      * elanTag. Important: ensure this method is only called whenever there is
1287      * no other ElanInterface in the specified DPN
1288      *
1289      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1290      */
1291     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1292             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1293         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1294         // last ElanInstance plus
1295         // adding a new one does (almost at the same time) are executed in that
1296         // exact order
1297
1298         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1299         FlowEntity flowEntity = new FlowEntityBuilder()
1300             .setDpnId(dpnId)
1301             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1302             .setFlowId(flowId)
1303             .build();
1304         mdsalManager.removeFlow(confTx, flowEntity);
1305     }
1306
1307     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1308             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1309         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1310         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1311     }
1312
1313     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1314             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1315         List<? extends MatchInfoBase> listMatchInfoBase;
1316         List<InstructionInfo> instructionInfos;
1317         long serviceId;
1318         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1319             serviceId = elanTag;
1320             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1321             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1322         } else {
1323             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1324             listMatchInfoBase = buildMatchesForVni(serviceId);
1325             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1326         }
1327         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1328                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1329                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1330                 instructionInfos);
1331         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1332     }
1333
1334     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1335             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1336         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1337         if (etreeInstance != null) {
1338             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1339         }
1340     }
1341
1342     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1343             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1344         long elanTag = elanInfo.getElanTag();
1345         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1346         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1347         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1348     }
1349
1350     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1351             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1352         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1353         if (etreeLeafTag != null) {
1354             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1355             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1356             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1357         }
1358     }
1359
1360     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1361             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1362         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1363                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1364                 5, elanInfo.getElanInstanceName(), 0, 0,
1365                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1366                 getMatchesForElanTag(elanTag, /* SH flag */false),
1367                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1368
1369         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1370     }
1371
1372     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1373             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1374         // only if ELAN can connect to external network, perform the following
1375
1376         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1377             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1378                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1379                     5, elanInfo.getElanInstanceName(), 0, 0,
1380                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1381                     getMatchesForElanTag(elanTag, /* SH flag */true),
1382                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1383             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1384         }
1385     }
1386
1387
1388     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1389             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1390             throws ExecutionException, InterruptedException {
1391         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1392                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1393         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1394
1395         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1396             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1397                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1398                     .build();
1399             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1400         }
1401     }
1402
1403     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1404         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1405     }
1406
1407     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1408             TypedWriteTransaction<Configuration> tx) {
1409         if (isStandardElanService(elanInterface)) {
1410             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1411                     elanInterface.getName(), lportTag, tx);
1412         } else { // Etree service
1413             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1414         }
1415     }
1416
1417     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1418             TypedWriteTransaction<Configuration> tx) {
1419         int instructionKey = 0;
1420         List<Instruction> instructions = new ArrayList<>();
1421         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1422                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1423
1424         List<Action> actions = new ArrayList<>();
1425         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1426                 lportTag).buildAction());
1427         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1428                 elanTag).buildAction());
1429         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1430
1431         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1432                 ++instructionKey));
1433
1434         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1435         BoundServices serviceInfo = ElanUtils.getBoundServices(
1436                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1437                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1438         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1439         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1440                 bindServiceId);
1441         if (!existingElanService.isPresent()) {
1442             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1443         }
1444     }
1445
1446     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1447             TypedWriteTransaction<Configuration> tx) {
1448         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1449             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1450                     lportTag, tx);
1451         } else {
1452             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1453             if (etreeInstance == null) {
1454                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1455                         elanInterface.getName(), elanInfo.getElanInstanceName());
1456             } else {
1457                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1458                         elanInterface.getName(), lportTag, tx);
1459             }
1460         }
1461     }
1462
1463     private boolean isStandardElanService(ElanInterface elanInterface) {
1464         return elanInterface.augmentation(EtreeInterface.class) == null;
1465     }
1466
1467     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1468             throws ExecutionException, InterruptedException {
1469         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1470         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1471         if (tx.read(bindServiceId).get().isPresent()) {
1472             tx.delete(bindServiceId);
1473         }
1474     }
1475
1476     private String getFlowRef(long tableId, long elanTag) {
1477         return String.valueOf(tableId) + elanTag;
1478     }
1479
1480     private String getFlowRef(long tableId, long elanTag, String flowName) {
1481         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1482                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1483     }
1484
1485     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1486         return String.valueOf(tableId) + elanTag + shFlag;
1487     }
1488
1489     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1490         List<Action> listAction = new ArrayList<>();
1491         int actionKey = 0;
1492         listAction.add(
1493             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1494         actionKey++;
1495         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1496         return listAction;
1497     }
1498
1499     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1500             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1501         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1502                 .withKey(new DpnInterfacesKey(dpId)).build();
1503         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1504                 CREATE_MISSING_PARENTS);
1505         return dpnInterface;
1506     }
1507
1508     /**
1509      * Delete elan dpn interface from operational DS.
1510      *
1511      * @param elanInstanceName
1512      *            the elan instance name
1513      * @param dpId
1514      *            the dp id
1515      */
1516     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1517             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1518         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1519                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1520         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1521         if (dpnInterfaces.isPresent()) {
1522             tx.delete(dpnInterfacesId);
1523         }
1524     }
1525
1526     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1527             TypedWriteTransaction<Operational> tx) {
1528         List<String> interfaceNames = new ArrayList<>();
1529         interfaceNames.add(interfaceName);
1530         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1531                 .withKey(new DpnInterfacesKey(dpId)).build();
1532         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1533                 CREATE_MISSING_PARENTS);
1534         return dpnInterface;
1535     }
1536
1537     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1538             throws ExecutionException, InterruptedException {
1539         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1540                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1541         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1542         // Adding new Elan Interface Port to the operational DataStore without
1543         // Static-Mac Entries..
1544         if (!interfaceMacTables.isPresent()) {
1545             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1546                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1547             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1548                     CREATE_MISSING_PARENTS);
1549         }
1550     }
1551
1552     private void createElanStateList(String elanInstanceName, String interfaceName,
1553             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1554         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1555         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1556         // Adding new Elan Interface Port to the operational DataStore without
1557         // Static-Mac Entries..
1558         if (elanInterfaceLists.isPresent()) {
1559             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1560             if (interfaceLists == null) {
1561                 interfaceLists = new ArrayList<>();
1562             }
1563             interfaceLists.add(interfaceName);
1564             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1565                     .withKey(new ElanKey(elanInstanceName)).build();
1566             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1567         }
1568     }
1569
1570     private boolean isOperational(InterfaceInfo interfaceInfo) {
1571         if (interfaceInfo == null) {
1572             return false;
1573         }
1574         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1575     }
1576
1577     @SuppressWarnings("checkstyle:IllegalCatch")
1578     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1579         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1580         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1581                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1582         if (dpnInterfaceLists == null) {
1583             return;
1584         }
1585         List<ElanDpnInterfacesList> elanDpnIf =
1586             requireNonNullElse(dpnInterfaceLists.getElanDpnInterfacesList(), emptyList());
1587         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1588             int cnt = 0;
1589             String elanName = elanDpns.getElanInstanceName();
1590             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1591             if (elanInfo == null) {
1592                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1593                         + "skipping this ELAN for tunnel handling", elanName);
1594                 continue;
1595             }
1596             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1597                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1598                 continue;
1599             }
1600             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1601             if (dpnInterfaces == null) {
1602                 continue;
1603             }
1604             DpnInterfaces dstDpnIf = null;
1605             for (DpnInterfaces dpnIf : dpnInterfaces) {
1606                 BigInteger dpnIfDpId = dpnIf.getDpId();
1607                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1608                     cnt++;
1609                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1610                     cnt++;
1611                     dstDpnIf = dpnIf;
1612                 }
1613             }
1614             if (cnt == 2) {
1615                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1616                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1617                 jobCoordinator.enqueueJob(elanName, () -> {
1618                     // update Remote BC Group
1619                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1620                     try {
1621                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1622                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1623                                 confTx)).get();
1624                     } catch (RuntimeException e) {
1625                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1626                     }
1627                     Set<String> interfaceLists = new HashSet<>();
1628                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1629                     for (String ifName : interfaceLists) {
1630                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1631                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1632                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1633                             if (isOperational(interfaceInfo)) {
1634                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1635                             }
1636                             return emptyList();
1637                         }, ElanConstants.JOB_MAX_RETRIES);
1638                     }
1639                     return emptyList();
1640                 }, ElanConstants.JOB_MAX_RETRIES);
1641             }
1642
1643         }
1644     }
1645
1646     /**
1647      * Handle external tunnel state event.
1648      *
1649      * @param externalTunnel
1650      *            the external tunnel
1651      * @param intrf
1652      *            the interface
1653      */
1654     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1655         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1656             return;
1657         }
1658         // dpId/externalNodeId will be available either in source or destination
1659         // based on the tunnel end point
1660         BigInteger dpId = null;
1661         NodeId externalNodeId = null;
1662         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1663             dpId = new BigInteger(externalTunnel.getSourceDevice());
1664             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1665         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1666             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1667             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1668         }
1669         if (dpId == null || externalNodeId == null) {
1670             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1671             return;
1672         }
1673
1674         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1675         if (dpnInterfaceLists == null) {
1676             return;
1677         }
1678         List<ElanDpnInterfacesList> elanDpnIf =
1679             requireNonNullElse(dpnInterfaceLists.getElanDpnInterfacesList(), emptyList());
1680         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1681             String elanName = elanDpns.getElanInstanceName();
1682             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1683
1684             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1685             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1686                     || dpnInterfaces.getInterfaces().isEmpty()) {
1687                 continue;
1688             }
1689             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1690
1691             final BigInteger finalDpId = dpId;
1692             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1693                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1694                 "Error setting up ELAN BGs");
1695             // install L2gwDevices local macs in dpn.
1696             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1697             // Install dpn macs on external device
1698             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1699                     externalNodeId);
1700         }
1701         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1702     }
1703
1704     /**
1705      * Installs dpn macs in external device. first it checks if the physical
1706      * locator towards this dpn tep is present or not if the physical locator is
1707      * present go ahead and add the ucast macs otherwise update the mcast mac
1708      * entry to include this dpn tep ip and schedule the job to put ucast macs
1709      * once the physical locator is programmed in device
1710      *
1711      * @param elanName
1712      *            the elan name
1713      * @param lstElanInterfaceNames
1714      *            the lst Elan interface names
1715      * @param dpnId
1716      *            the dpn id
1717      * @param externalNodeId
1718      *            the external node id
1719      */
1720     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1721             NodeId externalNodeId) {
1722         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1723                 externalNodeId.getValue());
1724         if (elanL2GwDevice == null) {
1725             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1726             return;
1727         }
1728         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1729         if (dpnTepIp == null) {
1730             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1731                     externalNodeId);
1732             return;
1733         }
1734
1735         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1736         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1737                 LogicalDatastoreType.OPERATIONAL);
1738         boolean phyLocAlreadyExists =
1739                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1740                 dpnTepIp);
1741         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1742                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1743         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1744
1745         if (phyLocAlreadyExists) {
1746             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1747             return;
1748         }
1749         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1750         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1751     }
1752
1753     /**
1754      * Validate external tunnel state event.
1755      *
1756      * @param externalTunnel
1757      *            the external tunnel
1758      * @param intrf
1759      *            the intrf
1760      * @return true, if successful
1761      */
1762     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1763         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1764             String srcDevice = externalTunnel.getDestinationDevice();
1765             String destDevice = externalTunnel.getSourceDevice();
1766             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1767                     LogicalDatastoreType.CONFIGURATION);
1768             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1769                     otherEndPointExtTunnel);
1770             if (otherEndPointExtTunnel != null) {
1771                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1772                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1773                 if (otherEndPointInterfaceOperational) {
1774                     return true;
1775                 } else {
1776                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1777                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1778                 }
1779             }
1780         }
1781         return false;
1782     }
1783
1784     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1785         List<MatchInfo> mkMatches = new ArrayList<>();
1786         // Matching metadata
1787         mkMatches.add(
1788                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1789         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1790         return mkMatches;
1791     }
1792
1793     @Override
1794     protected ElanInterfaceManager getDataTreeChangeListener() {
1795         return this;
1796     }
1797
1798     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1799                                              BigInteger dpId) {
1800         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1801         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1802             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1803                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1804         try {
1805             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1806         } catch (InterruptedException e) {
1807             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1808         }
1809     }
1810 }