elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.ListenableFuture;
  19 import java.math.BigInteger;
  20 import java.util.ArrayList;
  21 import java.util.Collections;
  22 import java.util.HashSet;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Objects;
  26 import java.util.Queue;
  27 import java.util.Set;
  28 import java.util.concurrent.ConcurrentHashMap;
  29 import java.util.concurrent.ConcurrentLinkedQueue;
  30 import java.util.concurrent.ExecutionException;
  31 import javax.annotation.PostConstruct;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.apache.commons.lang3.StringUtils;
  35 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  39 import org.opendaylight.genius.infra.Datastore.Configuration;
  40 import org.opendaylight.genius.infra.Datastore.Operational;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.infra.TransactionAdapter;
  44 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  45 import org.opendaylight.genius.infra.TypedWriteTransaction;
  46 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.itm.globals.ITMConstants;
  49 import org.opendaylight.genius.mdsalutil.FlowEntity;
  50 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  51 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  52 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  53 import org.opendaylight.genius.mdsalutil.MatchInfo;
  54 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  55 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  65 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  67 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  70 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  71 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  72 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  73 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  74 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  75 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  76 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  77 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  78 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  79 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  80 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  81 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  82 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  83 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  84 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  85 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  86 import org.opendaylight.serviceutils.srm.RecoverableListener;
  87 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  88 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  89 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  90 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 131 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 133 import org.slf4j.Logger;
 134 import org.slf4j.LoggerFactory;
 135
 136 /**
 137  * Class in charge of handling creations, modifications and removals of
 138  * ElanInterfaces.
 139  *
 140  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 141  */
 142 @Singleton
 143 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 144         implements RecoverableListener {
 145     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 146     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 147     private static final boolean SH_FLAG_SET = true;
 148     private static final boolean SH_FLAG_UNSET = false;
 149
 150     private final DataBroker broker;
 151     private final ManagedNewTransactionRunner txRunner;
 152     private final IMdsalApiManager mdsalManager;
 153     private final IInterfaceManager interfaceManager;
 154     private final IdManagerService idManager;
 155     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 156     private final INeutronVpnManager neutronVpnManager;
 157     private final ElanItmUtils elanItmUtils;
 158     private final ElanEtreeUtils elanEtreeUtils;
 159     private final ElanL2GatewayUtils elanL2GatewayUtils;
 160     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 161     private final ElanUtils elanUtils;
 162     private final JobCoordinator jobCoordinator;
 163     private final ElanInstanceCache elanInstanceCache;
 164     private final ElanInterfaceCache elanInterfaceCache;
 165
 166     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 167         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 168
 169     @Inject
 170     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 171                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 172                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 173                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 174                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 175                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 176                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 177                                 final ElanInstanceCache elanInstanceCache,
 178                                 final ElanInterfaceCache elanInterfaceCache,
 179                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 180                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 181         super(ElanInterface.class, ElanInterfaceManager.class);
 182         this.broker = dataBroker;
 183         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 184         this.idManager = managerService;
 185         this.mdsalManager = mdsalApiManager;
 186         this.interfaceManager = interfaceManager;
 187         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 188         this.neutronVpnManager = neutronVpnManager;
 189         this.elanItmUtils = elanItmUtils;
 190         this.elanEtreeUtils = elanEtreeUtils;
 191         this.elanL2GatewayUtils = elanL2GatewayUtils;
 192         this.elanUtils = elanUtils;
 193         this.jobCoordinator = jobCoordinator;
 194         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 195         this.elanInstanceCache = elanInstanceCache;
 196         this.elanInterfaceCache = elanInterfaceCache;
 197         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 198     }
 199
 200     @Override
 201     @PostConstruct
 202     public void init() {
 203         registerListener();
 204     }
 205
 206     @Override
 207     public void registerListener() {
 208         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 209     }
 210
 211     @Override
 212     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 213         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 214     }
 215
 216     @Override
 217     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 218         String interfaceName = del.getName();
 219         String elanInstanceName = del.getElanInstanceName();
 220         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 221         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 222             elanInterfaces.remove(del);
 223             if (elanInterfaces.isEmpty()) {
 224                 unProcessedElanInterfaces.remove(elanInstanceName);
 225             }
 226         }
 227         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 228         /*
 229          * Handling in case the elan instance is deleted.If the Elan instance is
 230          * deleted, there is no need to explicitly delete the elan interfaces
 231          */
 232         if (elanInfo == null) {
 233             return;
 234         }
 235         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 236         if (interfaceInfo == null && elanInfo.isExternal()) {
 237             // In deleting external network, the underlying ietf Inteface might have been removed
 238             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 239             // from Operational DS instead
 240             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 241         }
 242         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 243                 interfaceName, interfaceInfo, this);
 244         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 245     }
 246
 247     private static class RemoveElanInterfaceHolder {
 248         boolean isLastElanInterface = false;
 249         boolean isLastInterfaceOnDpn = false;
 250         BigInteger dpId = null;
 251     }
 252
 253     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 254     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 255             InterfaceInfo interfaceInfo) {
 256         String elanName = elanInfo.getElanInstanceName();
 257         long elanTag = elanInfo.getElanTag();
 258         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 259         List<ListenableFuture<Void>> futures = new ArrayList<>();
 260         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 261         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 262             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 263             if (elanState == null) {
 264                 return;
 265             }
 266             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 267                 List<String> elanInterfaces = elanState.getElanInterfaces();
 268                 if (elanInterfaces.isEmpty()) {
 269                     holder.isLastElanInterface = true;
 270                 }
 271                 if (interfaceInfo != null) {
 272                     holder.dpId = interfaceInfo.getDpId();
 273                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 274                         interfaceName, elanTag, interfaceTx);
 275                     /*
 276                      * If there are not elan ports, remove the unknown dmac, terminating
 277                      * service table flows, remote/local bc group
 278                      */
 279                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 280                         || dpnInterfaces.getInterfaces().isEmpty()) {
 281                         // No more Elan Interfaces in this DPN
 282                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 283                             holder.dpId);
 284                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 285                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 286                         }
 287                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 288                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 289                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 290                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 291                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 292                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 293                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 294                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 295                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 296                             }
 297                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 298                         }
 299                         holder.isLastInterfaceOnDpn = true;
 300                     } else {
 301                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 302                     }
 303                 }
 304             }));
 305         }));
 306         futures.forEach(ElanUtils::waitForTransactionToComplete);
 307
 308         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 309             futures.add(
 310                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 311                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 312         }
 313         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 314                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 315         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 316                 ElanConstants.JOB_MAX_RETRIES);
 317
 318         return futures;
 319     }
 320
 321     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 322             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 323             throws ExecutionException, InterruptedException {
 324         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 325         if (etreeLeafTag != null) {
 326             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 327             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 328         }
 329     }
 330
 331     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 332             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 333             throws ExecutionException, InterruptedException {
 334         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 335         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 336     }
 337
 338     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 339             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 340             throws ExecutionException, InterruptedException {
 341         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 342         if (etreeInstance != null) {
 343             BigInteger dpnId = interfaceInfo.getDpId();
 344             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 345             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 346             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 347         }
 348     }
 349
 350     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 351             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 352             throws ExecutionException, InterruptedException {
 353         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 354         if (etreeInstance != null) {
 355             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 356             BigInteger dpnId = interfaceInfo.getDpId();
 357             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 358             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 359             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 360         }
 361     }
 362
 363     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 364             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 365         String elanName = elanInfo.getElanInstanceName();
 366         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 367         if (elanState == null) {
 368             return elanState;
 369         }
 370         List<String> elanInterfaces = elanState.getElanInterfaces();
 371         boolean isRemoved = elanInterfaces.remove(interfaceName);
 372         if (!isRemoved) {
 373             return elanState;
 374         }
 375
 376         if (elanInterfaces.isEmpty()) {
 377             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 378             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 379             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 380         } else {
 381             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 382                     .withKey(new ElanKey(elanName)).build();
 383             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 384         }
 385         return elanState;
 386     }
 387
 388     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 389             throws ReadFailedException {
 390         // removing the ElanInterface from the config data_store if interface is
 391         // not present in Interface config DS
 392         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 393             interfaceName) == null
 394                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 395             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 396         }
 397     }
 398
 399     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 400             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 401         String elanName = elanInfo.getElanInstanceName();
 402         List<ListenableFuture<Void>> futures = new ArrayList<>();
 403         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 404             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 405                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 406                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 407                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 408                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 409                 if (interfaceInfo != null) {
 410                     if (existingElanInterfaceMac.isPresent()) {
 411                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 412                         if (existingMacEntries != null) {
 413                             List<PhysAddress> macAddresses = new ArrayList<>();
 414                             for (MacEntry macEntry : existingMacEntries) {
 415                                 PhysAddress macAddress = macEntry.getMacAddress();
 416                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 417                                         macAddress.getValue(), interfaceName);
 418                                 Optional<MacEntry> macEntryOptional =
 419                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 420                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 421                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 422                                 }
 423                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 424                                 macAddresses.add(macAddress);
 425                             }
 426
 427                             // Removing all those MACs from External Devices belonging
 428                             // to this ELAN
 429                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 430                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 431                             }
 432                         }
 433                     }
 434                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 435                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 436                 } else if (existingElanInterfaceMac.isPresent()) {
 437                     // Interface does not exist in ConfigDS, so lets remove everything
 438                     // about that interface related to Elan
 439                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 440                     if (macEntries != null) {
 441                         for (MacEntry macEntry : macEntries) {
 442                             PhysAddress macAddress = macEntry.getMacAddress();
 443                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 444                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 445                             }
 446                         }
 447                     }
 448                 }
 449                 if (existingElanInterfaceMac.isPresent()) {
 450                     interfaceTx.delete(elanInterfaceId);
 451                 }
 452                 unbindService(interfaceName, flowTx);
 453                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 454             }));
 455         }));
 456         return futures;
 457     }
 458
 459     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 460                                                                          String interfaceName, long elanTag,
 461                                                                          TypedReadWriteTransaction<Operational> tx)
 462             throws ExecutionException, InterruptedException {
 463         synchronized (elanName.intern()) {
 464
 465             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 466             if (dpnInterfaces != null) {
 467                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 468                 if (interfaceLists != null) {
 469                     interfaceLists.remove(interfaceName);
 470                 }
 471
 472                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 473                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 474                     deleteElanDpnInterface(elanName, dpId, tx);
 475                 } else {
 476                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 477                 }
 478             }
 479             return dpnInterfaces;
 480         }
 481     }
 482
 483     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 484         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 485         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 486             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 487                 BigInteger currentDpId = dpnInterface.getDpId();
 488                 if (!currentDpId.equals(dpId)) {
 489                     for (String elanInterface : dpnInterface.getInterfaces()) {
 490                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 491                         if (macs == null || macs.getMacEntry() == null) {
 492                             continue;
 493                         }
 494                         for (MacEntry mac : macs.getMacEntry()) {
 495                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 496                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 497                         }
 498                     }
 499                 }
 500             }
 501         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 502     }
 503
 504     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 505             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 506         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 507         if (etreeLeafTag != null) {
 508             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 509         }
 510     }
 511
 512     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 513             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 514         mdsalManager
 515                 .removeFlow(confTx, dpId,
 516                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 517                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 518                                         mac.getMacAddress().getValue(), elanTag)));
 519     }
 520
 521     /*
 522     * Possible Scenarios for update
 523     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 524         then all {1,2,3,4} should be removed
 525
 526         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 527         then all {1,2,3,4} should be added
 528
 529         c. if orig = {1,2,3,4} updated={2,3,4}
 530         then 1 should be removed
 531
 532         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 533         then we should just add 5
 534
 535         e. if orig = {1,2,3,4} updated={2,3,4,5}
 536         then 1 should be removed , 5 should be added
 537     * */
 538     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 539     @Override
 540     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 541         // updating the static-Mac Entries for the existing elanInterface
 542         String elanName = update.getElanInstanceName();
 543         String interfaceName = update.getName();
 544
 545         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 546         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 547         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 548         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 549
 550         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 551                 deletedEntry.getMacAddress()));
 552
 553         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 554         * Hence add the macentries for the same.*/
 555         for (StaticMacEntries staticMacEntry : updatedEntries) {
 556             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 557                     staticMacEntry.getMacAddress());
 558             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 559                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 560                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 561                     if (existingMacEntry.isPresent()) {
 562                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 563                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 564                             tx);
 565                     } else {
 566                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 567                             elanName, interfaceName, staticMacEntry, tx);
 568                     }
 569                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 570         }
 571     }
 572
 573     @Override
 574     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 575         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 576             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 577             String interfaceName = elanInterfaceAdded.getName();
 578             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 579             if (interfaceInfo == null) {
 580                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 581                 return;
 582             }
 583             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 584
 585             if (elanInstance == null) {
 586                 // Add the ElanInstance in the Configuration data-store
 587                 List<String> elanInterfaces = new ArrayList<>();
 588                 elanInterfaces.add(interfaceName);
 589                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 590                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 591                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 592                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 593             }
 594
 595             Long elanTag = elanInstance.getElanTag();
 596             // If elan tag is not updated, then put the elan interface into
 597             // unprocessed entry map and entry. Let entries
 598             // in this map get processed during ELAN update DCN.
 599             if (elanTag == null || elanTag == 0L) {
 600                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 601                 if (elanInterfaces == null) {
 602                     elanInterfaces = new ConcurrentLinkedQueue<>();
 603                 }
 604                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 605                     elanInterfaces.add(elanInterfaceAdded);
 606                 }
 607                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 608                 return;
 609             }
 610             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 611                     interfaceInfo, elanInstance, this);
 612             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 613         }), LOG, "Error processing added ELAN interface");
 614     }
 615
 616     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 617         List<ListenableFuture<Void>> futures = new ArrayList<>();
 618         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 619         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 620             return futures;
 621         }
 622         for (ElanInterface elanInterface : elanInterfaces) {
 623             String interfaceName = elanInterface.getName();
 624             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 625             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 626         }
 627         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 628         return futures;
 629     }
 630
 631     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 632             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 633         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 634                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 635         List<DpnInterfaces> dpnInterfaceLists = null;
 636         if (elanDpnInterfacesList != null) {
 637             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 638         }
 639         if (dpnInterfaceLists == null) {
 640             dpnInterfaceLists = new ArrayList<>();
 641         }
 642         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 643             BigInteger dstDpId = interfaceInfo.getDpId();
 644             if (dpnInterfaces.getDpId().equals(dstDpId)) {
 645                 continue;
 646             }
 647             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 648             for (String remoteIf : remoteElanInterfaces) {
 649                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 650                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 651                 if (elanIfMac == null || remoteInterface == null) {
 652                     continue;
 653                 }
 654                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 655                 if (remoteMacEntries != null) {
 656                     for (MacEntry macEntry : remoteMacEntries) {
 657                         String macAddress = macEntry.getMacAddress().getValue();
 658                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 659                                 dstDpId, elanInstance.getElanInstanceName());
 660                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 661                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 662                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 663                     }
 664                 }
 665             }
 666         }
 667     }
 668
 669     private static class AddElanInterfaceHolder {
 670         private DpnInterfaces dpnInterfaces = null;
 671         private boolean isFirstInterfaceInDpn = false;
 672         private BigInteger dpId;
 673     }
 674
 675     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 676     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 677             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 678         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 679         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 680         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 681
 682         String interfaceName = elanInterface.getName();
 683         String elanInstanceName = elanInterface.getElanInstanceName();
 684
 685         List<ListenableFuture<Void>> futures = new ArrayList<>();
 686         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 687         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 688             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 689             if (elanInfo == null) {
 690                 List<String> elanInterfaces = new ArrayList<>();
 691                 elanInterfaces.add(interfaceName);
 692                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 693                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 694                         operTx)));
 695             } else {
 696                 createElanStateList(elanInstanceName, interfaceName, operTx);
 697             }
 698             // Specific actions to the DPN where the ElanInterface has been added,
 699             // for example, programming the
 700             // External tunnel table if needed or adding the ElanInterface to the
 701             // DpnInterfaces in the operational DS.
 702             holder.dpId = interfaceInfo.getDpId();
 703             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 704                 synchronized (elanInstanceName.intern()) {
 705                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 706                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 707                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 708                     if (ElanUtils.isVlan(elanInstance)) {
 709                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 710                             elanInstanceName, existingElanDpnInterfaces);
 711                     } else {
 712                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 713                     }
 714                     if (holder.isFirstInterfaceInDpn) {
 715                         // ELAN's 1st ElanInterface added to this DPN
 716                         if (!existingElanDpnInterfaces.isPresent()) {
 717                             holder.dpnInterfaces =
 718                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 719                         } else {
 720                             List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 721                             elanInterfaces.add(interfaceName);
 722                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 723                                 elanInterfaces, operTx);
 724                         }
 725                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 726                         // table, but only if Elan has VNI
 727                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 728                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 729                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 730                         }
 731                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 732                             interfaceName);
 733                     } else {
 734                         List<String> elanInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 735                         elanInterfaces.add(interfaceName);
 736                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 737                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 738                                 interfaceName);
 739                         }
 740                         holder.dpnInterfaces =
 741                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 742                     }
 743                 }
 744             }
 745
 746             // add code to install Local/Remote BC group, unknow DMAC entry,
 747             // terminating service table flow entry
 748             // call bindservice of interfacemanager to create ingress table flow
 749             // enty.
 750             // Add interface to the ElanInterfaceForwardingEntires Container
 751             createElanInterfaceTablesList(interfaceName, operTx);
 752         }));
 753         futures.forEach(ElanUtils::waitForTransactionToComplete);
 754         futures.add(
 755             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 756                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 757                     holder.isFirstInterfaceInDpn, confTx))));
 758
 759         // add the vlan provider interface to remote BC group for the elan
 760         // for internal vlan networks
 761         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 762             if (interfaceManager.isExternalInterface(interfaceName)) {
 763                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 764                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 765             }
 766         }
 767
 768         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 769             //update the remote-DPNs remoteBC group entry with Tunnels
 770             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 771                 holder.dpId);
 772             futures.add(
 773                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 774                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 775         }
 776
 777         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 778         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 779                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 780         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 781         return futures;
 782     }
 783
 784     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 785     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 786             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 787         String elanInstanceName = elanInstance.getElanInstanceName();
 788         String interfaceName = elanInterface.getName();
 789         List<ListenableFuture<Void>> futures = new ArrayList<>();
 790         BigInteger dpId = interfaceInfo.getDpId();
 791         boolean isInterfaceOperational = isOperational(interfaceInfo);
 792         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 793             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 794                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 795                     isFirstInterfaceInDpn, confTx, operTx);
 796
 797                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 798                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 799
 800                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 801                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 802                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 803                             staticMacEntry.getMacAddress());
 804                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 805                             LogicalDatastoreType.OPERATIONAL, macId);
 806                         if (existingMacEntry.isPresent()) {
 807                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 808                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 809                                 existingMacEntry.get(), operTx);
 810                         } else {
 811                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 812                                 interfaceName, staticMacEntry, operTx);
 813                         }
 814
 815                         if (isInterfaceOperational) {
 816                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 817                             // DPNs
 818                             String macAddress = staticMacEntry.getMacAddress().getValue();
 819                             LOG.info(
 820                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 821                                     + " {}",
 822                                 macAddress, elanInstanceName, interfaceName);
 823                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 824                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 825                         }
 826                     }
 827
 828                     if (isInterfaceOperational) {
 829                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 830                         // on purpose.
 831                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 832                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 833                         }
 834                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 835                             staticMacAddresses);
 836                     }
 837                 }
 838             }));
 839         }));
 840         futures.forEach(ElanUtils::waitForTransactionToComplete);
 841         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 842             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 843             try {
 844                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 845                 if (neutronPort != null) {
 846                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 847                 }
 848             } catch (IllegalArgumentException ex) {
 849                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 850             }
 851         }
 852         return futures;
 853     }
 854
 855     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 856             PhysAddress physAddress) {
 857         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 858         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 859         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 860                 LogicalDatastoreType.OPERATIONAL, macId);
 861
 862         if (!existingMacEntry.isPresent()) {
 863             return;
 864         }
 865
 866         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 867                 .withKey(new MacEntryKey(physAddress)).build();
 868         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 869                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 870     }
 871
 872     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 873             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 874         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 875         if (!existingElanDpnInterfaces.isPresent()) {
 876             return true;
 877         }
 878         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 879             return false;
 880         }
 881         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 882         int dummyInterfaceCount =  0;
 883         if (dpnInterfaces.getInterfaces().contains(routerPortUuid)) {
 884             dummyInterfaceCount++;
 885         }
 886         if (dpnInterfaces.getInterfaces().contains(elanInstanceName)) {
 887             dummyInterfaceCount++;
 888         }
 889         return dpnInterfaces.getInterfaces().size() - dummyInterfaceCount == 0;
 890     }
 891
 892     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 893         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 894                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 895     }
 896
 897     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 898             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx,
 899             TypedWriteTransaction<Operational> operTx) {
 900         if (!isOperational(interfaceInfo)) {
 901             return;
 902         }
 903         BigInteger dpId = interfaceInfo.getDpId();
 904         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 905             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 906         }
 907         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 908         if (isFirstInterfaceInDpn) {
 909             // Terminating Service , UnknownDMAC Table.
 910             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 911             // for ELAN Instance is VxLAN
 912             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 913                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 914             }
 915             setupUnknownDMacTable(elanInstance, dpId, confTx);
 916             /*
 917              * Install remote DMAC flow. This is required since this DPN is
 918              * added later to the elan instance and remote DMACs of other
 919              * interfaces in this elan instance are not present in the current
 920              * dpn.
 921              */
 922             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 923                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 924                         elanInstance.getElanInstanceName());
 925                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 926             }
 927         }
 928         // bind the Elan service to the Interface
 929         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 930     }
 931
 932     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 933             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 934         if (!isOperational(interfaceInfo)) {
 935             return;
 936         }
 937         // LocalBroadcast Group creation with elan-Interfaces
 938         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 939         if (isFirstInterfaceInDpn) {
 940             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 941             BigInteger dpId = interfaceInfo.getDpId();
 942             // RemoteBroadcast Group creation
 943             try {
 944                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 945             } catch (InterruptedException e1) {
 946                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 947             }
 948             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 949             try {
 950                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 951             } catch (InterruptedException e1) {
 952                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 953             }
 954         }
 955     }
 956
 957     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 958             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 959         int ifTag = interfaceInfo.getInterfaceTag();
 960         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 961                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 962                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 963                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 964                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 965
 966         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 967
 968         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 969                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 970                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 971                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 972
 973         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 974     }
 975
 976     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 977             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 978         int ifTag = interfaceInfo.getInterfaceTag();
 979         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 980             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
 981
 982         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
 983
 984         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 985             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 986             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 987             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 988
 989         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
 990     }
 991
 992     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
 993             TypedWriteTransaction<Configuration> confTx) {
 994         int elanTag = elanInfo.getElanTag().intValue();
 995         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
 996         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
 997         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 998         if (etreeInstance != null) {
 999             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1000             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1001             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1002         }
1003     }
1004
1005     @SuppressWarnings("checkstyle:IllegalCatch")
1006     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1007             TypedWriteTransaction<Configuration> confTx) {
1008         int bucketId = 0;
1009         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1010         if (elanDpns != null) {
1011             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1012             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1013                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1014                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1015                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1016                     List<Action> listAction = new ArrayList<>();
1017                     int actionKey = 0;
1018                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1019                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1020                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1021                     bucketId++;
1022                     for (DpnInterfaces otherFes : dpnInterfaces) {
1023                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1024                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1025                             && !otherFes.getInterfaces().isEmpty()) {
1026                             try {
1027                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1028                                         dpnInterface.getDpId(), otherFes.getDpId(),
1029                                         elanUtils.isOpenstackVniSemanticsEnforced()
1030                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1031                                 if (!remoteListActionInfo.isEmpty()) {
1032                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1033                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1034                                     bucketId++;
1035                                 }
1036                             } catch (Exception ex) {
1037                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1038                                         + "Logical Group Interface not found between source Dpn - {}, "
1039                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1040                                 return;
1041                             }
1042                         }
1043                     }
1044                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1045                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1046                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1047
1048                     if (remoteListBucketInfo.isEmpty()) {
1049                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1050                         continue;
1051                     }
1052                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1053                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1054                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1055                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1056                 }
1057             }
1058             try {
1059                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1060             } catch (InterruptedException e1) {
1061                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1062             }
1063         }
1064     }
1065
1066     private List<MatchInfo> buildMatchesForVni(Long vni) {
1067         List<MatchInfo> mkMatches = new ArrayList<>();
1068         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1069         mkMatches.add(match);
1070         return mkMatches;
1071     }
1072
1073     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1074         List<InstructionInfo> mkInstructions = new ArrayList<>();
1075         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1076         return mkInstructions;
1077     }
1078
1079     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1080         List<MatchInfo> mkMatches = new ArrayList<>();
1081         // Matching metadata
1082         mkMatches.add(new MatchMetadata(
1083                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1084         return mkMatches;
1085     }
1086
1087     /**
1088      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1089      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1090      *
1091      * @param elanTag
1092      *            elanTag to be written in metadata when flow is selected
1093      * @return the instructions ready to be installed in a flow
1094      */
1095     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1096         List<InstructionInfo> mkInstructions = new ArrayList<>();
1097         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1098                 .getElanMetadataMask()));
1099         /* applicable for EXTERNAL_TUNNEL_TABLE only
1100         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1101         */
1102         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1103         return mkInstructions;
1104     }
1105
1106     // Install DMAC entry on dst DPN
1107     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1108             BigInteger dstDpId) {
1109         String interfaceName = interfaceInfo.getInterfaceName();
1110         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1111         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1112             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1113             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1114                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1115                     for (MacEntry macEntry : macEntries) {
1116                         String macAddress = macEntry.getMacAddress().getValue();
1117                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1118                             interfaceName);
1119                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1120                             interfaceInfo.getDpId())) {
1121                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1122                                 macAddress);
1123                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1124                         }
1125                     }
1126                 })));
1127         }
1128         return Collections.emptyList();
1129     }
1130
1131     private void createDropBucket(List<Bucket> listBucket) {
1132         List<Action> actionsInfos = new ArrayList<>();
1133         actionsInfos.add(new ActionDrop().buildAction());
1134         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1135                 MDSALUtil.WATCH_GROUP);
1136         listBucket.add(dropBucket);
1137     }
1138
1139     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1140             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1141         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1142         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1143     }
1144
1145     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1146             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1147         List<Bucket> listBucket = new ArrayList<>();
1148         int bucketId = 0;
1149         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1150
1151         List<String> interfaces = new ArrayList<>();
1152         if (newDpnInterface != null) {
1153             interfaces = newDpnInterface.getInterfaces();
1154         }
1155         for (String ifName : interfaces) {
1156             // In case if there is a InterfacePort in the cache which is not in
1157             // operational state, skip processing it
1158             InterfaceInfo ifInfo = interfaceManager
1159                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1160             if (!isOperational(ifInfo)) {
1161                 continue;
1162             }
1163
1164             if (!interfaceManager.isExternalInterface(ifName)) {
1165                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1166                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1167                 bucketId++;
1168             }
1169         }
1170
1171         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1172                 MDSALUtil.buildBucketLists(listBucket));
1173         LOG.trace("installing the localBroadCast Group:{}", group);
1174         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1175     }
1176
1177     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1178             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1179         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1180         if (etreeInstance != null) {
1181             List<Bucket> listBucket = new ArrayList<>();
1182             int bucketId = 0;
1183
1184             List<String> interfaces = new ArrayList<>();
1185             if (newDpnInterface != null) {
1186                 interfaces = newDpnInterface.getInterfaces();
1187             }
1188             for (String ifName : interfaces) {
1189                 // In case if there is a InterfacePort in the cache which is not
1190                 // in
1191                 // operational state, skip processing it
1192                 InterfaceInfo ifInfo = interfaceManager
1193                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1194                 if (!isOperational(ifInfo)) {
1195                     continue;
1196                 }
1197
1198                 if (!interfaceManager.isExternalInterface(ifName)) {
1199                     // only add root interfaces
1200                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1201                 }
1202             }
1203
1204             if (listBucket.isEmpty()) { // No Buckets
1205                 createDropBucket(listBucket);
1206             }
1207
1208             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1209             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1210             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1211                     MDSALUtil.buildBucketLists(listBucket));
1212             LOG.trace("installing the localBroadCast Group:{}", group);
1213             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1214         }
1215     }
1216
1217     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1218             InterfaceInfo ifInfo) {
1219         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1220         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1221             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1222                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1223             bucketId++;
1224         }
1225         return bucketId;
1226     }
1227
1228     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1229             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1230             throws ExecutionException, InterruptedException {
1231         BigInteger dpnId = interfaceInfo.getDpId();
1232         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1233         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1234         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1235     }
1236
1237     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1238             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1239             throws ExecutionException, InterruptedException {
1240         BigInteger dpnId = interfaceInfo.getDpId();
1241         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1242         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1243         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1244     }
1245
1246     /**
1247      * Installs a flow in the External Tunnel table consisting in translating
1248      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1249      * elanTag that will be used later in the ELANs pipeline.
1250      * @param dpnId the dpn id
1251      */
1252     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1253             TypedWriteTransaction<Configuration> confTx) {
1254         long elanTag = elanInfo.getElanTag();
1255         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1256                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1257                 elanInfo.getElanInstanceName(), // flowName
1258                 0, // idleTimeout
1259                 0, // hardTimeout
1260                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1261                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1262                 getInstructionsIntOrExtTunnelTable(elanTag));
1263
1264         mdsalManager.addFlow(confTx, flowEntity);
1265     }
1266
1267     /**
1268      * Removes, from External Tunnel table, the flow that translates from VNI to
1269      * elanTag. Important: ensure this method is only called whenever there is
1270      * no other ElanInterface in the specified DPN
1271      *
1272      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1273      */
1274     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1275             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1276         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1277         // last ElanInstance plus
1278         // adding a new one does (almost at the same time) are executed in that
1279         // exact order
1280
1281         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1282         FlowEntity flowEntity = new FlowEntityBuilder()
1283             .setDpnId(dpnId)
1284             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1285             .setFlowId(flowId)
1286             .build();
1287         mdsalManager.removeFlow(confTx, flowEntity);
1288     }
1289
1290     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1291             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1292         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1293         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1294     }
1295
1296     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1297             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1298         List<? extends MatchInfoBase> listMatchInfoBase;
1299         List<InstructionInfo> instructionInfos;
1300         long serviceId;
1301         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1302             serviceId = elanTag;
1303             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1304             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1305         } else {
1306             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1307             listMatchInfoBase = buildMatchesForVni(serviceId);
1308             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1309         }
1310         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1311                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1312                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1313                 instructionInfos);
1314         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1315     }
1316
1317     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1318             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1319         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1320         if (etreeInstance != null) {
1321             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1322         }
1323     }
1324
1325     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1326             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1327         long elanTag = elanInfo.getElanTag();
1328         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1329         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1330         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1331     }
1332
1333     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1334             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1335         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1336         if (etreeLeafTag != null) {
1337             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1338             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1339             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1340         }
1341     }
1342
1343     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1344             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1345         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1346                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1347                 5, elanInfo.getElanInstanceName(), 0, 0,
1348                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1349                 getMatchesForElanTag(elanTag, /* SH flag */false),
1350                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1351
1352         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1353     }
1354
1355     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1356             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1357         // only if ELAN can connect to external network, perform the following
1358
1359         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1360             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1361                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1362                     5, elanInfo.getElanInstanceName(), 0, 0,
1363                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1364                     getMatchesForElanTag(elanTag, /* SH flag */true),
1365                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1366             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1367         }
1368     }
1369
1370
1371     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1372             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1373             throws ExecutionException, InterruptedException {
1374         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1375                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1376         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1377
1378         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1379             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1380                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1381                     .build();
1382             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1383         }
1384     }
1385
1386     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1387         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1388     }
1389
1390     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1391             TypedWriteTransaction<Configuration> tx) {
1392         if (isStandardElanService(elanInterface)) {
1393             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1394                     elanInterface.getName(), lportTag, tx);
1395         } else { // Etree service
1396             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1397         }
1398     }
1399
1400     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1401             TypedWriteTransaction<Configuration> tx) {
1402         int instructionKey = 0;
1403         List<Instruction> instructions = new ArrayList<>();
1404         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1405                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1406
1407         List<Action> actions = new ArrayList<>();
1408         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1409                 lportTag).buildAction());
1410         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1411                 elanTag).buildAction());
1412         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1413
1414         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1415                 ++instructionKey));
1416
1417         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1418         BoundServices serviceInfo = ElanUtils.getBoundServices(
1419                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1420                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1421         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1422         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1423                 bindServiceId);
1424         if (!existingElanService.isPresent()) {
1425             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1426         }
1427     }
1428
1429     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1430             TypedWriteTransaction<Configuration> tx) {
1431         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1432             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1433                     lportTag, tx);
1434         } else {
1435             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1436             if (etreeInstance == null) {
1437                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1438                         elanInterface.getName(), elanInfo.getElanInstanceName());
1439             } else {
1440                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1441                         elanInterface.getName(), lportTag, tx);
1442             }
1443         }
1444     }
1445
1446     private boolean isStandardElanService(ElanInterface elanInterface) {
1447         return elanInterface.augmentation(EtreeInterface.class) == null;
1448     }
1449
1450     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1451             throws ExecutionException, InterruptedException {
1452         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1453         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1454         if (tx.read(bindServiceId).get().isPresent()) {
1455             tx.delete(bindServiceId);
1456         }
1457     }
1458
1459     private String getFlowRef(long tableId, long elanTag) {
1460         return String.valueOf(tableId) + elanTag;
1461     }
1462
1463     private String getFlowRef(long tableId, long elanTag, String flowName) {
1464         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1465                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1466     }
1467
1468     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1469         return String.valueOf(tableId) + elanTag + shFlag;
1470     }
1471
1472     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1473         List<Action> listAction = new ArrayList<>();
1474         int actionKey = 0;
1475         listAction.add(
1476             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1477         actionKey++;
1478         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1479         return listAction;
1480     }
1481
1482     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1483             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1484         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1485                 .withKey(new DpnInterfacesKey(dpId)).build();
1486         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1487                 CREATE_MISSING_PARENTS);
1488         return dpnInterface;
1489     }
1490
1491     /**
1492      * Delete elan dpn interface from operational DS.
1493      *
1494      * @param elanInstanceName
1495      *            the elan instance name
1496      * @param dpId
1497      *            the dp id
1498      */
1499     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1500             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1501         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1502                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1503         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1504         if (dpnInterfaces.isPresent()) {
1505             tx.delete(dpnInterfacesId);
1506         }
1507     }
1508
1509     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1510             TypedWriteTransaction<Operational> tx) {
1511         List<String> interfaceNames = new ArrayList<>();
1512         interfaceNames.add(interfaceName);
1513         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1514                 .withKey(new DpnInterfacesKey(dpId)).build();
1515         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1516                 CREATE_MISSING_PARENTS);
1517         return dpnInterface;
1518     }
1519
1520     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1521             throws ExecutionException, InterruptedException {
1522         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1523                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1524         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1525         // Adding new Elan Interface Port to the operational DataStore without
1526         // Static-Mac Entries..
1527         if (!interfaceMacTables.isPresent()) {
1528             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1529                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1530             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1531                     CREATE_MISSING_PARENTS);
1532         }
1533     }
1534
1535     private void createElanStateList(String elanInstanceName, String interfaceName,
1536             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1537         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1538         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1539         // Adding new Elan Interface Port to the operational DataStore without
1540         // Static-Mac Entries..
1541         if (elanInterfaceLists.isPresent()) {
1542             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1543             if (interfaceLists == null) {
1544                 interfaceLists = new ArrayList<>();
1545             }
1546             interfaceLists.add(interfaceName);
1547             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1548                     .withKey(new ElanKey(elanInstanceName)).build();
1549             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1550         }
1551     }
1552
1553     private boolean isOperational(InterfaceInfo interfaceInfo) {
1554         if (interfaceInfo == null) {
1555             return false;
1556         }
1557         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1558     }
1559
1560     @SuppressWarnings("checkstyle:IllegalCatch")
1561     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1562         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1563         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1564                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1565         if (dpnInterfaceLists == null) {
1566             return;
1567         }
1568         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1569         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1570             int cnt = 0;
1571             String elanName = elanDpns.getElanInstanceName();
1572             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1573             if (elanInfo == null) {
1574                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1575                         + "skipping this ELAN for tunnel handling", elanName);
1576                 continue;
1577             }
1578             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1579                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1580                 continue;
1581             }
1582             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1583             if (dpnInterfaces == null) {
1584                 continue;
1585             }
1586             DpnInterfaces dstDpnIf = null;
1587             for (DpnInterfaces dpnIf : dpnInterfaces) {
1588                 BigInteger dpnIfDpId = dpnIf.getDpId();
1589                 if (dpnIfDpId.equals(srcDpId)) {
1590                     cnt++;
1591                 } else if (dpnIfDpId.equals(dstDpId)) {
1592                     cnt++;
1593                     dstDpnIf = dpnIf;
1594                 }
1595             }
1596             if (cnt == 2) {
1597                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1598                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1599                 jobCoordinator.enqueueJob(elanName, () -> {
1600                     // update Remote BC Group
1601                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1602                     try {
1603                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1604                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1605                                 confTx)).get();
1606                     } catch (RuntimeException e) {
1607                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1608                     }
1609                     Set<String> interfaceLists = new HashSet<>();
1610                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1611                     for (String ifName : interfaceLists) {
1612                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1613                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1614                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1615                             if (isOperational(interfaceInfo)) {
1616                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1617                             }
1618                             return Collections.emptyList();
1619                         }, ElanConstants.JOB_MAX_RETRIES);
1620                     }
1621                     return Collections.emptyList();
1622                 }, ElanConstants.JOB_MAX_RETRIES);
1623             }
1624
1625         }
1626     }
1627
1628     /**
1629      * Handle external tunnel state event.
1630      *
1631      * @param externalTunnel
1632      *            the external tunnel
1633      * @param intrf
1634      *            the interface
1635      */
1636     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1637         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1638             return;
1639         }
1640         // dpId/externalNodeId will be available either in source or destination
1641         // based on the tunnel end point
1642         BigInteger dpId = null;
1643         NodeId externalNodeId = null;
1644         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1645             dpId = new BigInteger(externalTunnel.getSourceDevice());
1646             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1647         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1648             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1649             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1650         }
1651         if (dpId == null || externalNodeId == null) {
1652             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1653             return;
1654         }
1655
1656         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1657         if (dpnInterfaceLists == null) {
1658             return;
1659         }
1660         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.getElanDpnInterfacesList();
1661         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1662             String elanName = elanDpns.getElanInstanceName();
1663             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1664
1665             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1666             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1667                     || dpnInterfaces.getInterfaces().isEmpty()) {
1668                 continue;
1669             }
1670             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1671
1672             final BigInteger finalDpId = dpId;
1673             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1674                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1675                 "Error setting up ELAN BGs");
1676             // install L2gwDevices local macs in dpn.
1677             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1678             // Install dpn macs on external device
1679             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1680                     externalNodeId);
1681         }
1682         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1683     }
1684
1685     /**
1686      * Installs dpn macs in external device. first it checks if the physical
1687      * locator towards this dpn tep is present or not if the physical locator is
1688      * present go ahead and add the ucast macs otherwise update the mcast mac
1689      * entry to include this dpn tep ip and schedule the job to put ucast macs
1690      * once the physical locator is programmed in device
1691      *
1692      * @param elanName
1693      *            the elan name
1694      * @param lstElanInterfaceNames
1695      *            the lst Elan interface names
1696      * @param dpnId
1697      *            the dpn id
1698      * @param externalNodeId
1699      *            the external node id
1700      */
1701     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1702             NodeId externalNodeId) {
1703         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1704                 externalNodeId.getValue());
1705         if (elanL2GwDevice == null) {
1706             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1707             return;
1708         }
1709         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1710         if (dpnTepIp == null) {
1711             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1712                     externalNodeId);
1713             return;
1714         }
1715
1716         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1717         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1718                 LogicalDatastoreType.OPERATIONAL);
1719         boolean phyLocAlreadyExists =
1720                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1721                 dpnTepIp);
1722         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1723                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1724         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1725
1726         if (phyLocAlreadyExists) {
1727             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1728             return;
1729         }
1730         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1731         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1732     }
1733
1734     /**
1735      * Validate external tunnel state event.
1736      *
1737      * @param externalTunnel
1738      *            the external tunnel
1739      * @param intrf
1740      *            the intrf
1741      * @return true, if successful
1742      */
1743     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1744         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1745             String srcDevice = externalTunnel.getDestinationDevice();
1746             String destDevice = externalTunnel.getSourceDevice();
1747             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1748                     LogicalDatastoreType.CONFIGURATION);
1749             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1750                     otherEndPointExtTunnel);
1751             if (otherEndPointExtTunnel != null) {
1752                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1753                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1754                 if (otherEndPointInterfaceOperational) {
1755                     return true;
1756                 } else {
1757                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1758                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1759                 }
1760             }
1761         }
1762         return false;
1763     }
1764
1765     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1766         List<MatchInfo> mkMatches = new ArrayList<>();
1767         // Matching metadata
1768         mkMatches.add(
1769                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1770         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1771         return mkMatches;
1772     }
1773
1774     @Override
1775     protected ElanInterfaceManager getDataTreeChangeListener() {
1776         return this;
1777     }
1778
1779     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1780                                              BigInteger dpId) {
1781         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1782         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1783             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1784                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1785         try {
1786             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1787         } catch (InterruptedException e) {
1788             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1789         }
1790     }
1791 }