elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Lists;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Queue;
  28 import java.util.Set;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.locks.ReentrantLock;
  33 import javax.annotation.PostConstruct;
  34 import javax.inject.Inject;
  35 import javax.inject.Singleton;
  36 import org.apache.commons.lang3.StringUtils;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  40 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  41 import org.opendaylight.genius.infra.Datastore.Configuration;
  42 import org.opendaylight.genius.infra.Datastore.Operational;
  43 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  45 import org.opendaylight.genius.infra.TransactionAdapter;
  46 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  47 import org.opendaylight.genius.infra.TypedWriteTransaction;
  48 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  49 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  50 import org.opendaylight.genius.itm.globals.ITMConstants;
  51 import org.opendaylight.genius.mdsalutil.FlowEntity;
  52 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  53 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  54 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  55 import org.opendaylight.genius.mdsalutil.MatchInfo;
  56 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  57 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  64 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  67 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  68 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  70 import org.opendaylight.genius.utils.JvmGlobalLocks;
  71 import org.opendaylight.genius.utils.ServiceIndex;
  72 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  73 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  74 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  75 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  76 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  78 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  79 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  80 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  81 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  82 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  84 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  85 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  86 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  87 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  88 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  89 import org.opendaylight.serviceutils.srm.RecoverableListener;
  90 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  91 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 134 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 135 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 /**
 140  * Class in charge of handling creations, modifications and removals of
 141  * ElanInterfaces.
 142  *
 143  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 144  */
 145 @Singleton
 146 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 147         implements RecoverableListener {
 148     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 149     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 150     private static final boolean SH_FLAG_SET = true;
 151     private static final boolean SH_FLAG_UNSET = false;
 152
 153     private final DataBroker broker;
 154     private final ManagedNewTransactionRunner txRunner;
 155     private final IMdsalApiManager mdsalManager;
 156     private final IInterfaceManager interfaceManager;
 157     private final IdManagerService idManager;
 158     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 159     private final INeutronVpnManager neutronVpnManager;
 160     private final ElanItmUtils elanItmUtils;
 161     private final ElanEtreeUtils elanEtreeUtils;
 162     private final ElanL2GatewayUtils elanL2GatewayUtils;
 163     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 164     private final ElanUtils elanUtils;
 165     private final JobCoordinator jobCoordinator;
 166     private final ElanInstanceCache elanInstanceCache;
 167     private final ElanInterfaceCache elanInterfaceCache;
 168
 169     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 170         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 171
 172     @Inject
 173     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 174                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 175                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 176                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 177                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 178                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 179                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 180                                 final ElanInstanceCache elanInstanceCache,
 181                                 final ElanInterfaceCache elanInterfaceCache,
 182                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 183                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 184         super(ElanInterface.class, ElanInterfaceManager.class);
 185         this.broker = dataBroker;
 186         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 187         this.idManager = managerService;
 188         this.mdsalManager = mdsalApiManager;
 189         this.interfaceManager = interfaceManager;
 190         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 191         this.neutronVpnManager = neutronVpnManager;
 192         this.elanItmUtils = elanItmUtils;
 193         this.elanEtreeUtils = elanEtreeUtils;
 194         this.elanL2GatewayUtils = elanL2GatewayUtils;
 195         this.elanUtils = elanUtils;
 196         this.jobCoordinator = jobCoordinator;
 197         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 198         this.elanInstanceCache = elanInstanceCache;
 199         this.elanInterfaceCache = elanInterfaceCache;
 200         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 201     }
 202
 203     @Override
 204     @PostConstruct
 205     public void init() {
 206         registerListener();
 207     }
 208
 209     @Override
 210     public void registerListener() {
 211         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 212     }
 213
 214     @Override
 215     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 216         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 217     }
 218
 219     @Override
 220     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 221         String interfaceName = del.getName();
 222         String elanInstanceName = del.getElanInstanceName();
 223         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 224         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 225             elanInterfaces.remove(del);
 226             if (elanInterfaces.isEmpty()) {
 227                 unProcessedElanInterfaces.remove(elanInstanceName);
 228             }
 229         }
 230         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 231         /*
 232          * Handling in case the elan instance is deleted.If the Elan instance is
 233          * deleted, there is no need to explicitly delete the elan interfaces
 234          */
 235         if (elanInfo == null) {
 236             return;
 237         }
 238         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 239         if (interfaceInfo == null && elanInfo.isExternal()) {
 240             // In deleting external network, the underlying ietf Inteface might have been removed
 241             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 242             // from Operational DS instead
 243             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 244         }
 245         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 246                 interfaceName, interfaceInfo, this);
 247         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 248     }
 249
 250     private static class RemoveElanInterfaceHolder {
 251         boolean isLastElanInterface = false;
 252         boolean isLastInterfaceOnDpn = false;
 253         BigInteger dpId = null;
 254     }
 255
 256     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 257     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 258             InterfaceInfo interfaceInfo) {
 259         String elanName = elanInfo.getElanInstanceName();
 260         long elanTag = elanInfo.getElanTag();
 261         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 262         List<ListenableFuture<Void>> futures = new ArrayList<>();
 263         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 264         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 265             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 266             if (elanState == null) {
 267                 return;
 268             }
 269             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 270                 List<String> elanInterfaces = elanState.getElanInterfaces();
 271                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 272                     holder.isLastElanInterface = true;
 273                 }
 274                 if (interfaceInfo != null) {
 275                     holder.dpId = interfaceInfo.getDpId();
 276                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 277                         interfaceName, elanTag, interfaceTx);
 278                     /*
 279                      * If there are not elan ports, remove the unknown dmac, terminating
 280                      * service table flows, remote/local bc group
 281                      */
 282                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 283                         || dpnInterfaces.getInterfaces().isEmpty()) {
 284                         // No more Elan Interfaces in this DPN
 285                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 286                             holder.dpId);
 287                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 288                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 289                         }
 290                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 291                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 292                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 293                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 295                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 296                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 297                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 298                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 299                             }
 300                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 301                         }
 302                         holder.isLastInterfaceOnDpn = true;
 303                     } else {
 304                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 305                     }
 306                 }
 307             }));
 308         }));
 309         futures.forEach(ElanUtils::waitForTransactionToComplete);
 310
 311         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 312             futures.add(
 313                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 314                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 315         }
 316         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 317                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 318         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 319                 ElanConstants.JOB_MAX_RETRIES);
 320
 321         return futures;
 322     }
 323
 324     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 325             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 326             throws ExecutionException, InterruptedException {
 327         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 328         if (etreeLeafTag != null) {
 329             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 330             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 331         }
 332     }
 333
 334     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 335             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 336             throws ExecutionException, InterruptedException {
 337         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 338         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339     }
 340
 341     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 342             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 343             throws ExecutionException, InterruptedException {
 344         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 345         if (etreeInstance != null) {
 346             BigInteger dpnId = interfaceInfo.getDpId();
 347             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 348             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 349             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 350         }
 351     }
 352
 353     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 354             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 355             throws ExecutionException, InterruptedException {
 356         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 357         if (etreeInstance != null) {
 358             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 359             BigInteger dpnId = interfaceInfo.getDpId();
 360             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 361             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 362             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 363         }
 364     }
 365
 366     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 367             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 368         String elanName = elanInfo.getElanInstanceName();
 369         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 370         if (elanState == null) {
 371             return elanState;
 372         }
 373         List<String> elanInterfaces = elanState.getElanInterfaces();
 374         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 375         if (!isRemoved) {
 376             return elanState;
 377         }
 378
 379         if (elanInterfaces.isEmpty()) {
 380             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 381             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 383         } else {
 384             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 385                     .withKey(new ElanKey(elanName)).build();
 386             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 387         }
 388         return elanState;
 389     }
 390
 391     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 392             throws ReadFailedException {
 393         // removing the ElanInterface from the config data_store if interface is
 394         // not present in Interface config DS
 395         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 396             interfaceName) == null
 397                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 398             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 399         }
 400     }
 401
 402     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 403             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 404         String elanName = elanInfo.getElanInstanceName();
 405         List<ListenableFuture<Void>> futures = new ArrayList<>();
 406         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 407             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 408                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 409                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 410                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 411                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 412                 if (interfaceInfo != null) {
 413                     if (existingElanInterfaceMac.isPresent()) {
 414                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 415                         if (existingMacEntries != null) {
 416                             List<PhysAddress> macAddresses = new ArrayList<>();
 417                             for (MacEntry macEntry : existingMacEntries) {
 418                                 PhysAddress macAddress = macEntry.getMacAddress();
 419                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 420                                         macAddress.getValue(), interfaceName);
 421                                 Optional<MacEntry> macEntryOptional =
 422                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 423                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 424                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 425                                 }
 426                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 427                                 macAddresses.add(macAddress);
 428                             }
 429
 430                             // Removing all those MACs from External Devices belonging
 431                             // to this ELAN
 432                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 433                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 434                             }
 435                         }
 436                     }
 437                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 438                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 439                 } else if (existingElanInterfaceMac.isPresent()) {
 440                     // Interface does not exist in ConfigDS, so lets remove everything
 441                     // about that interface related to Elan
 442                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 443                     if (macEntries != null) {
 444                         for (MacEntry macEntry : macEntries) {
 445                             PhysAddress macAddress = macEntry.getMacAddress();
 446                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 447                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 448                             }
 449                         }
 450                     }
 451                 }
 452                 if (existingElanInterfaceMac.isPresent()) {
 453                     interfaceTx.delete(elanInterfaceId);
 454                 }
 455                 unbindService(interfaceName, flowTx);
 456                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 457             }));
 458         }));
 459         return futures;
 460     }
 461
 462     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 463                                                                          String interfaceName, long elanTag,
 464                                                                          TypedReadWriteTransaction<Operational> tx)
 465             throws ExecutionException, InterruptedException {
 466         // FIXME: pass in and use ElanInstanceKey instead?
 467         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 468         lock.lock();
 469         try {
 470             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 471             if (dpnInterfaces != null) {
 472                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 473                 if (interfaceLists != null) {
 474                     interfaceLists.remove(interfaceName);
 475                 }
 476
 477                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 478                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 479                     deleteElanDpnInterface(elanName, dpId, tx);
 480                 } else {
 481                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 482                 }
 483             }
 484             return dpnInterfaces;
 485         } finally {
 486             lock.unlock();
 487         }
 488     }
 489
 490     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 491         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 492         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 493             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 494                 BigInteger currentDpId = dpnInterface.getDpId();
 495                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 496                     for (String elanInterface : dpnInterface.getInterfaces()) {
 497                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 498                         if (macs == null || macs.getMacEntry() == null) {
 499                             continue;
 500                         }
 501                         for (MacEntry mac : macs.getMacEntry()) {
 502                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 503                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 504                         }
 505                     }
 506                 }
 507             }
 508         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 509     }
 510
 511     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 512             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 513         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 514         if (etreeLeafTag != null) {
 515             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 516         }
 517     }
 518
 519     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 520             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 521         mdsalManager
 522                 .removeFlow(confTx, dpId,
 523                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 524                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 525                                         mac.getMacAddress().getValue(), elanTag)));
 526     }
 527
 528     /*
 529     * Possible Scenarios for update
 530     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 531         then all {1,2,3,4} should be removed
 532
 533         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 534         then all {1,2,3,4} should be added
 535
 536         c. if orig = {1,2,3,4} updated={2,3,4}
 537         then 1 should be removed
 538
 539         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 540         then we should just add 5
 541
 542         e. if orig = {1,2,3,4} updated={2,3,4,5}
 543         then 1 should be removed , 5 should be added
 544     * */
 545     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 546     @Override
 547     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 548         // updating the static-Mac Entries for the existing elanInterface
 549         String elanName = update.getElanInstanceName();
 550         String interfaceName = update.getName();
 551
 552         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 553         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 554         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 555         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 556
 557         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 558                 deletedEntry.getMacAddress()));
 559
 560         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 561         * Hence add the macentries for the same.*/
 562         for (StaticMacEntries staticMacEntry : updatedEntries) {
 563             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 564                     staticMacEntry.getMacAddress());
 565             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 566                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 567                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 568                     if (existingMacEntry.isPresent()) {
 569                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 570                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 571                             tx);
 572                     } else {
 573                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 574                             elanName, interfaceName, staticMacEntry, tx);
 575                     }
 576                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 577         }
 578     }
 579
 580     @Override
 581     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 582         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 583             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 584             String interfaceName = elanInterfaceAdded.getName();
 585             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 586             if (interfaceInfo == null) {
 587                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 588                 return;
 589             }
 590             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 591
 592             if (elanInstance == null) {
 593                 // Add the ElanInstance in the Configuration data-store
 594                 List<String> elanInterfaces = new ArrayList<>();
 595                 elanInterfaces.add(interfaceName);
 596                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 597                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 598                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 599                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 600             }
 601
 602             Long elanTag = elanInstance.getElanTag();
 603             // If elan tag is not updated, then put the elan interface into
 604             // unprocessed entry map and entry. Let entries
 605             // in this map get processed during ELAN update DCN.
 606             if (elanTag == null || elanTag == 0L) {
 607                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 608                 if (elanInterfaces == null) {
 609                     elanInterfaces = new ConcurrentLinkedQueue<>();
 610                 }
 611                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 612                     elanInterfaces.add(elanInterfaceAdded);
 613                 }
 614                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 615                 return;
 616             }
 617             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 618                     interfaceInfo, elanInstance, this);
 619             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 620         }), LOG, "Error processing added ELAN interface");
 621     }
 622
 623     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 624         List<ListenableFuture<Void>> futures = new ArrayList<>();
 625         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 626         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 627             return futures;
 628         }
 629         for (ElanInterface elanInterface : elanInterfaces) {
 630             String interfaceName = elanInterface.getName();
 631             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 632             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 633         }
 634         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 635         return futures;
 636     }
 637
 638     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 639             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 640         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 641                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 642         List<DpnInterfaces> dpnInterfaceLists = null;
 643         if (elanDpnInterfacesList != null) {
 644             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 645         }
 646         if (dpnInterfaceLists == null) {
 647             dpnInterfaceLists = new ArrayList<>();
 648         }
 649         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 650             BigInteger dstDpId = interfaceInfo.getDpId();
 651             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 652                 continue;
 653             }
 654             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 655             for (String remoteIf : remoteElanInterfaces) {
 656                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 657                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 658                 if (elanIfMac == null || remoteInterface == null) {
 659                     continue;
 660                 }
 661                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 662                 if (remoteMacEntries != null) {
 663                     for (MacEntry macEntry : remoteMacEntries) {
 664                         String macAddress = macEntry.getMacAddress().getValue();
 665                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 666                                 dstDpId, elanInstance.getElanInstanceName());
 667                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 668                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 669                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 670                     }
 671                 }
 672             }
 673         }
 674     }
 675
 676     private static class AddElanInterfaceHolder {
 677         private DpnInterfaces dpnInterfaces = null;
 678         private boolean isFirstInterfaceInDpn = false;
 679         private BigInteger dpId;
 680     }
 681
 682     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 683     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 684             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 685         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 686         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 687         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 688
 689         String interfaceName = elanInterface.getName();
 690         String elanInstanceName = elanInterface.getElanInstanceName();
 691
 692         List<ListenableFuture<Void>> futures = new ArrayList<>();
 693         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 694         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 695             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 696             if (elanInfo == null) {
 697                 List<String> elanInterfaces = new ArrayList<>();
 698                 elanInterfaces.add(interfaceName);
 699                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 700                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 701                         operTx)));
 702             } else {
 703                 createElanStateList(elanInstanceName, interfaceName, operTx);
 704             }
 705             // Specific actions to the DPN where the ElanInterface has been added,
 706             // for example, programming the
 707             // External tunnel table if needed or adding the ElanInterface to the
 708             // DpnInterfaces in the operational DS.
 709             holder.dpId = interfaceInfo.getDpId();
 710             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 711                 // FIXME: use elanInstaince.key() instead?
 712                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 713                 lock.lock();
 714                 try {
 715                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 716                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 717                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 718                     if (ElanUtils.isVlan(elanInstance)) {
 719                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 720                             elanInstanceName, existingElanDpnInterfaces);
 721                     } else {
 722                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 723                     }
 724                     if (holder.isFirstInterfaceInDpn) {
 725                         // ELAN's 1st ElanInterface added to this DPN
 726                         if (!existingElanDpnInterfaces.isPresent()) {
 727                             holder.dpnInterfaces =
 728                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 729                         } else {
 730                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 731                             List<String> elanInterfaces =
 732                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 733                             elanInterfaces.add(interfaceName);
 734                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 735                                 elanInterfaces, operTx);
 736                         }
 737                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 738                         // table, but only if Elan has VNI
 739                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 740                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 741                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 742                         }
 743                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 744                             interfaceName);
 745                     } else {
 746                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 747                         List<String> elanInterfaces =
 748                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 749                         elanInterfaces.add(interfaceName);
 750                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 751                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 752                                 interfaceName);
 753                         }
 754                         holder.dpnInterfaces =
 755                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 756                     }
 757                 } finally {
 758                     lock.unlock();
 759                 }
 760             }
 761
 762             // add code to install Local/Remote BC group, unknow DMAC entry,
 763             // terminating service table flow entry
 764             // call bindservice of interfacemanager to create ingress table flow
 765             // enty.
 766             // Add interface to the ElanInterfaceForwardingEntires Container
 767             createElanInterfaceTablesList(interfaceName, operTx);
 768         }));
 769         futures.forEach(ElanUtils::waitForTransactionToComplete);
 770         futures.add(
 771             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 772                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 773                     holder.isFirstInterfaceInDpn, confTx))));
 774
 775         // add the vlan provider interface to remote BC group for the elan
 776         // for internal vlan networks
 777         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 778             if (interfaceManager.isExternalInterface(interfaceName)) {
 779                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 780                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 781             }
 782         }
 783
 784         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 785             //update the remote-DPNs remoteBC group entry with Tunnels
 786             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 787                 holder.dpId);
 788             futures.add(
 789                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 790                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 791         }
 792
 793         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 794         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 795                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 796         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 797         return futures;
 798     }
 799
 800     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 801     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 802             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 803         String elanInstanceName = elanInstance.getElanInstanceName();
 804         String interfaceName = elanInterface.getName();
 805         List<ListenableFuture<Void>> futures = new ArrayList<>();
 806         BigInteger dpId = interfaceInfo.getDpId();
 807         boolean isInterfaceOperational = isOperational(interfaceInfo);
 808         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 809             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 810                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 811                     isFirstInterfaceInDpn, confTx);
 812
 813                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 814                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 815
 816                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 817                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 818                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 819                             staticMacEntry.getMacAddress());
 820                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 821                             LogicalDatastoreType.OPERATIONAL, macId);
 822                         if (existingMacEntry.isPresent()) {
 823                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 824                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 825                                 existingMacEntry.get(), operTx);
 826                         } else {
 827                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 828                                 interfaceName, staticMacEntry, operTx);
 829                         }
 830
 831                         if (isInterfaceOperational) {
 832                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 833                             // DPNs
 834                             String macAddress = staticMacEntry.getMacAddress().getValue();
 835                             LOG.info(
 836                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 837                                     + " {}",
 838                                 macAddress, elanInstanceName, interfaceName);
 839                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 840                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 841                         }
 842                     }
 843
 844                     if (isInterfaceOperational) {
 845                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 846                         // on purpose.
 847                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 848                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 849                         }
 850                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 851                             staticMacAddresses);
 852                     }
 853                 }
 854             }));
 855         }));
 856         futures.forEach(ElanUtils::waitForTransactionToComplete);
 857         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 858             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 859             try {
 860                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 861                 if (neutronPort != null) {
 862                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 863                 }
 864             } catch (IllegalArgumentException ex) {
 865                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 866             }
 867         }
 868         return futures;
 869     }
 870
 871     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 872             PhysAddress physAddress) {
 873         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 874         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 875         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 876                 LogicalDatastoreType.OPERATIONAL, macId);
 877
 878         if (!existingMacEntry.isPresent()) {
 879             return;
 880         }
 881
 882         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 883                 .withKey(new MacEntryKey(physAddress)).build();
 884         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 885                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 886     }
 887
 888     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 889             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 890         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 891         if (!existingElanDpnInterfaces.isPresent()) {
 892             return true;
 893         }
 894         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 895             return false;
 896         }
 897         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 898         int dummyInterfaceCount =  0;
 899         List<String> interfaces = dpnInterfaces.getInterfaces();
 900         if (interfaces == null) {
 901             return true;
 902         }
 903         if (interfaces.contains(routerPortUuid)) {
 904             dummyInterfaceCount++;
 905         }
 906         if (interfaces.contains(elanInstanceName)) {
 907             dummyInterfaceCount++;
 908         }
 909         return interfaces.size() == dummyInterfaceCount;
 910     }
 911
 912     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 913         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 914                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 915     }
 916
 917     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 918             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 919         if (!isOperational(interfaceInfo)) {
 920             return;
 921         }
 922         BigInteger dpId = interfaceInfo.getDpId();
 923         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 924             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 925         }
 926         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 927         if (isFirstInterfaceInDpn) {
 928             // Terminating Service , UnknownDMAC Table.
 929             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 930             // for ELAN Instance is VxLAN
 931             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 932                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 933             }
 934             setupUnknownDMacTable(elanInstance, dpId, confTx);
 935             /*
 936              * Install remote DMAC flow. This is required since this DPN is
 937              * added later to the elan instance and remote DMACs of other
 938              * interfaces in this elan instance are not present in the current
 939              * dpn.
 940              */
 941             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 942                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 943                         elanInstance.getElanInstanceName());
 944                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 945             }
 946         }
 947         // bind the Elan service to the Interface
 948         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 949     }
 950
 951     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 952             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 953         if (!isOperational(interfaceInfo)) {
 954             return;
 955         }
 956         // LocalBroadcast Group creation with elan-Interfaces
 957         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 958         if (isFirstInterfaceInDpn) {
 959             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 960             BigInteger dpId = interfaceInfo.getDpId();
 961             // RemoteBroadcast Group creation
 962             try {
 963                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 964             } catch (InterruptedException e1) {
 965                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 966             }
 967             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 968             try {
 969                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 970             } catch (InterruptedException e1) {
 971                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 972             }
 973         }
 974     }
 975
 976     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 977             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 978         int ifTag = interfaceInfo.getInterfaceTag();
 979         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 980                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 981                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 982                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 983                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 984
 985         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 986
 987         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 988                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 989                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 990                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 991
 992         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 993     }
 994
 995     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 996             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 997         int ifTag = interfaceInfo.getInterfaceTag();
 998         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 999             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1000
1001         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1002
1003         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1004             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1005             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1006             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1007
1008         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1009     }
1010
1011     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1012             TypedWriteTransaction<Configuration> confTx) {
1013         int elanTag = elanInfo.getElanTag().intValue();
1014         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1015         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1016         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1017         if (etreeInstance != null) {
1018             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1019             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1020             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1021         }
1022     }
1023
1024     @SuppressWarnings("checkstyle:IllegalCatch")
1025     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1026             TypedWriteTransaction<Configuration> confTx) {
1027         int bucketId = 0;
1028         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1029         if (elanDpns != null) {
1030             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1031             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1032                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1033                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1034                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1035                     List<Action> listAction = new ArrayList<>();
1036                     int actionKey = 0;
1037                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1038                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1039                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1040                     bucketId++;
1041                     for (DpnInterfaces otherFes : dpnInterfaces) {
1042                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1043                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1044                             && !otherFes.getInterfaces().isEmpty()) {
1045                             try {
1046                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1047                                         dpnInterface.getDpId(), otherFes.getDpId(),
1048                                         elanUtils.isOpenstackVniSemanticsEnforced()
1049                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1050                                 if (!remoteListActionInfo.isEmpty()) {
1051                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1052                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1053                                     bucketId++;
1054                                 }
1055                             } catch (Exception ex) {
1056                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1057                                         + "Logical Group Interface not found between source Dpn - {}, "
1058                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1059                                 return;
1060                             }
1061                         }
1062                     }
1063                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1064                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1065                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1066
1067                     if (remoteListBucketInfo.isEmpty()) {
1068                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1069                         continue;
1070                     }
1071                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1072                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1073                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1074                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1075                 }
1076             }
1077             try {
1078                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1079             } catch (InterruptedException e1) {
1080                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1081             }
1082         }
1083     }
1084
1085     private List<MatchInfo> buildMatchesForVni(Long vni) {
1086         List<MatchInfo> mkMatches = new ArrayList<>();
1087         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1088         mkMatches.add(match);
1089         return mkMatches;
1090     }
1091
1092     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1093         List<InstructionInfo> mkInstructions = new ArrayList<>();
1094         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1095         return mkInstructions;
1096     }
1097
1098     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1099         List<MatchInfo> mkMatches = new ArrayList<>();
1100         // Matching metadata
1101         mkMatches.add(new MatchMetadata(
1102                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1103         return mkMatches;
1104     }
1105
1106     /**
1107      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1108      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1109      *
1110      * @param elanTag
1111      *            elanTag to be written in metadata when flow is selected
1112      * @return the instructions ready to be installed in a flow
1113      */
1114     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1115         List<InstructionInfo> mkInstructions = new ArrayList<>();
1116         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1117                 .getElanMetadataMask()));
1118         /* applicable for EXTERNAL_TUNNEL_TABLE only
1119         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1120         */
1121         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1122         return mkInstructions;
1123     }
1124
1125     // Install DMAC entry on dst DPN
1126     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1127             BigInteger dstDpId) {
1128         String interfaceName = interfaceInfo.getInterfaceName();
1129         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1130         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1131             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1132             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1133                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1134                     for (MacEntry macEntry : macEntries) {
1135                         String macAddress = macEntry.getMacAddress().getValue();
1136                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1137                             interfaceName);
1138                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1139                             interfaceInfo.getDpId())) {
1140                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1141                                 macAddress);
1142                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1143                         }
1144                     }
1145                 })));
1146         }
1147         return emptyList();
1148     }
1149
1150     private void createDropBucket(List<Bucket> listBucket) {
1151         List<Action> actionsInfos = new ArrayList<>();
1152         actionsInfos.add(new ActionDrop().buildAction());
1153         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1154                 MDSALUtil.WATCH_GROUP);
1155         listBucket.add(dropBucket);
1156     }
1157
1158     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1159             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1160         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1161         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1162     }
1163
1164     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1165             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1166         List<Bucket> listBucket = new ArrayList<>();
1167         int bucketId = 0;
1168         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1169
1170         List<String> interfaces = new ArrayList<>();
1171         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1172             interfaces = newDpnInterface.getInterfaces();
1173         }
1174         for (String ifName : interfaces) {
1175             // In case if there is a InterfacePort in the cache which is not in
1176             // operational state, skip processing it
1177             InterfaceInfo ifInfo = interfaceManager
1178                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1179             if (!isOperational(ifInfo)) {
1180                 continue;
1181             }
1182
1183             if (!interfaceManager.isExternalInterface(ifName)) {
1184                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1185                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1186                 bucketId++;
1187             }
1188         }
1189
1190         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1191                 MDSALUtil.buildBucketLists(listBucket));
1192         LOG.trace("installing the localBroadCast Group:{}", group);
1193         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1194     }
1195
1196     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1197             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1198         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1199         if (etreeInstance != null) {
1200             List<Bucket> listBucket = new ArrayList<>();
1201             int bucketId = 0;
1202
1203             List<String> interfaces = new ArrayList<>();
1204             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1205                 interfaces = newDpnInterface.getInterfaces();
1206             }
1207             for (String ifName : interfaces) {
1208                 // In case if there is a InterfacePort in the cache which is not
1209                 // in
1210                 // operational state, skip processing it
1211                 InterfaceInfo ifInfo = interfaceManager
1212                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1213                 if (!isOperational(ifInfo)) {
1214                     continue;
1215                 }
1216
1217                 if (!interfaceManager.isExternalInterface(ifName)) {
1218                     // only add root interfaces
1219                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1220                 }
1221             }
1222
1223             if (listBucket.isEmpty()) { // No Buckets
1224                 createDropBucket(listBucket);
1225             }
1226
1227             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1228             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1229             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1230                     MDSALUtil.buildBucketLists(listBucket));
1231             LOG.trace("installing the localBroadCast Group:{}", group);
1232             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1233         }
1234     }
1235
1236     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1237             InterfaceInfo ifInfo) {
1238         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1239         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1240             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1241                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1242             bucketId++;
1243         }
1244         return bucketId;
1245     }
1246
1247     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1248             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1249             throws ExecutionException, InterruptedException {
1250         BigInteger dpnId = interfaceInfo.getDpId();
1251         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1252         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1253         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1254     }
1255
1256     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1257             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1258             throws ExecutionException, InterruptedException {
1259         BigInteger dpnId = interfaceInfo.getDpId();
1260         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1261         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1262         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1263     }
1264
1265     /**
1266      * Installs a flow in the External Tunnel table consisting in translating
1267      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1268      * elanTag that will be used later in the ELANs pipeline.
1269      * @param dpnId the dpn id
1270      */
1271     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1272             TypedWriteTransaction<Configuration> confTx) {
1273         long elanTag = elanInfo.getElanTag();
1274         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1275                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1276                 elanInfo.getElanInstanceName(), // flowName
1277                 0, // idleTimeout
1278                 0, // hardTimeout
1279                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1280                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1281                 getInstructionsIntOrExtTunnelTable(elanTag));
1282
1283         mdsalManager.addFlow(confTx, flowEntity);
1284     }
1285
1286     /**
1287      * Removes, from External Tunnel table, the flow that translates from VNI to
1288      * elanTag. Important: ensure this method is only called whenever there is
1289      * no other ElanInterface in the specified DPN
1290      *
1291      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1292      */
1293     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1294             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1295         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1296         // last ElanInstance plus
1297         // adding a new one does (almost at the same time) are executed in that
1298         // exact order
1299
1300         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1301         FlowEntity flowEntity = new FlowEntityBuilder()
1302             .setDpnId(dpnId)
1303             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1304             .setFlowId(flowId)
1305             .build();
1306         mdsalManager.removeFlow(confTx, flowEntity);
1307     }
1308
1309     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1310             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1311         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1312         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1313     }
1314
1315     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1316             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1317         List<? extends MatchInfoBase> listMatchInfoBase;
1318         List<InstructionInfo> instructionInfos;
1319         long serviceId;
1320         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1321             serviceId = elanTag;
1322             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1323             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1324         } else {
1325             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1326             listMatchInfoBase = buildMatchesForVni(serviceId);
1327             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1328         }
1329         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1330                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1331                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1332                 instructionInfos);
1333         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1334     }
1335
1336     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1337             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1338         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1339         if (etreeInstance != null) {
1340             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1341         }
1342     }
1343
1344     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1345             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1346         long elanTag = elanInfo.getElanTag();
1347         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1348         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1349         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1350     }
1351
1352     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1353             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1354         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1355         if (etreeLeafTag != null) {
1356             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1357             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1358             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1359         }
1360     }
1361
1362     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1363             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1364         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1365                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1366                 5, elanInfo.getElanInstanceName(), 0, 0,
1367                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1368                 getMatchesForElanTag(elanTag, /* SH flag */false),
1369                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1370
1371         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1372     }
1373
1374     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1375             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1376         // only if ELAN can connect to external network, perform the following
1377
1378         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1379             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1380                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1381                     5, elanInfo.getElanInstanceName(), 0, 0,
1382                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1383                     getMatchesForElanTag(elanTag, /* SH flag */true),
1384                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1385             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1386         }
1387     }
1388
1389
1390     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1391             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1392             throws ExecutionException, InterruptedException {
1393         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1394                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1395         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1396
1397         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1398             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1399                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1400                     .build();
1401             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1402         }
1403     }
1404
1405     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1406         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1407     }
1408
1409     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1410             TypedWriteTransaction<Configuration> tx) {
1411         if (isStandardElanService(elanInterface)) {
1412             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1413                     elanInterface.getName(), lportTag, tx);
1414         } else { // Etree service
1415             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1416         }
1417     }
1418
1419     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1420             TypedWriteTransaction<Configuration> tx) {
1421         int instructionKey = 0;
1422         List<Instruction> instructions = new ArrayList<>();
1423         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1424                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1425
1426         List<Action> actions = new ArrayList<>();
1427         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1428                 lportTag).buildAction());
1429         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1430                 elanTag).buildAction());
1431         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1432
1433         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1434                 ++instructionKey));
1435
1436         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1437         BoundServices serviceInfo = ElanUtils.getBoundServices(
1438                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1439                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1440         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1441         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1442                 bindServiceId);
1443         if (!existingElanService.isPresent()) {
1444             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1445         }
1446     }
1447
1448     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1449             TypedWriteTransaction<Configuration> tx) {
1450         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1451             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1452                     lportTag, tx);
1453         } else {
1454             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1455             if (etreeInstance == null) {
1456                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1457                         elanInterface.getName(), elanInfo.getElanInstanceName());
1458             } else {
1459                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1460                         elanInterface.getName(), lportTag, tx);
1461             }
1462         }
1463     }
1464
1465     private boolean isStandardElanService(ElanInterface elanInterface) {
1466         return elanInterface.augmentation(EtreeInterface.class) == null;
1467     }
1468
1469     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1470             throws ExecutionException, InterruptedException {
1471         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1472         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1473         if (tx.read(bindServiceId).get().isPresent()) {
1474             tx.delete(bindServiceId);
1475         }
1476     }
1477
1478     private String getFlowRef(long tableId, long elanTag) {
1479         return String.valueOf(tableId) + elanTag;
1480     }
1481
1482     private String getFlowRef(long tableId, long elanTag, String flowName) {
1483         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1484                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1485     }
1486
1487     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1488         return String.valueOf(tableId) + elanTag + shFlag;
1489     }
1490
1491     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1492         List<Action> listAction = new ArrayList<>();
1493         int actionKey = 0;
1494         listAction.add(
1495             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1496         actionKey++;
1497         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1498         return listAction;
1499     }
1500
1501     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1502             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1503         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1504                 .withKey(new DpnInterfacesKey(dpId)).build();
1505         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1506                 CREATE_MISSING_PARENTS);
1507         return dpnInterface;
1508     }
1509
1510     /**
1511      * Delete elan dpn interface from operational DS.
1512      *
1513      * @param elanInstanceName
1514      *            the elan instance name
1515      * @param dpId
1516      *            the dp id
1517      */
1518     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1519             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1520         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1521                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1522         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1523         if (dpnInterfaces.isPresent()) {
1524             tx.delete(dpnInterfacesId);
1525         }
1526     }
1527
1528     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1529             TypedWriteTransaction<Operational> tx) {
1530         List<String> interfaceNames = new ArrayList<>();
1531         interfaceNames.add(interfaceName);
1532         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1533                 .withKey(new DpnInterfacesKey(dpId)).build();
1534         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1535                 CREATE_MISSING_PARENTS);
1536         return dpnInterface;
1537     }
1538
1539     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1540             throws ExecutionException, InterruptedException {
1541         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1542                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1543         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1544         // Adding new Elan Interface Port to the operational DataStore without
1545         // Static-Mac Entries..
1546         if (!interfaceMacTables.isPresent()) {
1547             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1548                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1549             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1550                     CREATE_MISSING_PARENTS);
1551         }
1552     }
1553
1554     private void createElanStateList(String elanInstanceName, String interfaceName,
1555             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1556         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1557         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1558         // Adding new Elan Interface Port to the operational DataStore without
1559         // Static-Mac Entries..
1560         if (elanInterfaceLists.isPresent()) {
1561             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1562             if (interfaceLists == null) {
1563                 interfaceLists = new ArrayList<>();
1564             }
1565             interfaceLists.add(interfaceName);
1566             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1567                     .withKey(new ElanKey(elanInstanceName)).build();
1568             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1569         }
1570     }
1571
1572     private boolean isOperational(InterfaceInfo interfaceInfo) {
1573         if (interfaceInfo == null) {
1574             return false;
1575         }
1576         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1577     }
1578
1579     @SuppressWarnings("checkstyle:IllegalCatch")
1580     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1581         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1582         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1583                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1584         if (dpnInterfaceLists == null) {
1585             return;
1586         }
1587         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1588         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1589             int cnt = 0;
1590             String elanName = elanDpns.getElanInstanceName();
1591             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1592             if (elanInfo == null) {
1593                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1594                         + "skipping this ELAN for tunnel handling", elanName);
1595                 continue;
1596             }
1597             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1598                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1599                 continue;
1600             }
1601             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1602             if (dpnInterfaces == null) {
1603                 continue;
1604             }
1605             DpnInterfaces dstDpnIf = null;
1606             for (DpnInterfaces dpnIf : dpnInterfaces) {
1607                 BigInteger dpnIfDpId = dpnIf.getDpId();
1608                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1609                     cnt++;
1610                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1611                     cnt++;
1612                     dstDpnIf = dpnIf;
1613                 }
1614             }
1615             if (cnt == 2) {
1616                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1617                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1618                 jobCoordinator.enqueueJob(elanName, () -> {
1619                     // update Remote BC Group
1620                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1621                     try {
1622                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1623                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1624                                 confTx)).get();
1625                     } catch (RuntimeException e) {
1626                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1627                     }
1628                     Set<String> interfaceLists = new HashSet<>();
1629                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1630                     for (String ifName : interfaceLists) {
1631                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1632                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1633                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1634                             if (isOperational(interfaceInfo)) {
1635                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1636                             }
1637                             return emptyList();
1638                         }, ElanConstants.JOB_MAX_RETRIES);
1639                     }
1640                     return emptyList();
1641                 }, ElanConstants.JOB_MAX_RETRIES);
1642             }
1643
1644         }
1645     }
1646
1647     /**
1648      * Handle external tunnel state event.
1649      *
1650      * @param externalTunnel
1651      *            the external tunnel
1652      * @param intrf
1653      *            the interface
1654      */
1655     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1656         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1657             return;
1658         }
1659         // dpId/externalNodeId will be available either in source or destination
1660         // based on the tunnel end point
1661         BigInteger dpId = null;
1662         NodeId externalNodeId = null;
1663         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1664             dpId = new BigInteger(externalTunnel.getSourceDevice());
1665             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1666         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1667             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1668             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1669         }
1670         if (dpId == null || externalNodeId == null) {
1671             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1672             return;
1673         }
1674
1675         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1676         if (dpnInterfaceLists == null) {
1677             return;
1678         }
1679         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1680         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1681             String elanName = elanDpns.getElanInstanceName();
1682             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1683
1684             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1685             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1686                     || dpnInterfaces.getInterfaces().isEmpty()) {
1687                 continue;
1688             }
1689             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1690
1691             final BigInteger finalDpId = dpId;
1692             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1693                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1694                 "Error setting up ELAN BGs");
1695             // install L2gwDevices local macs in dpn.
1696             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1697             // Install dpn macs on external device
1698             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1699                     externalNodeId);
1700         }
1701         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1702     }
1703
1704     /**
1705      * Installs dpn macs in external device. first it checks if the physical
1706      * locator towards this dpn tep is present or not if the physical locator is
1707      * present go ahead and add the ucast macs otherwise update the mcast mac
1708      * entry to include this dpn tep ip and schedule the job to put ucast macs
1709      * once the physical locator is programmed in device
1710      *
1711      * @param elanName
1712      *            the elan name
1713      * @param lstElanInterfaceNames
1714      *            the lst Elan interface names
1715      * @param dpnId
1716      *            the dpn id
1717      * @param externalNodeId
1718      *            the external node id
1719      */
1720     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1721             NodeId externalNodeId) {
1722         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1723                 externalNodeId.getValue());
1724         if (elanL2GwDevice == null) {
1725             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1726             return;
1727         }
1728         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1729         if (dpnTepIp == null) {
1730             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1731                     externalNodeId);
1732             return;
1733         }
1734
1735         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1736         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1737                 LogicalDatastoreType.OPERATIONAL);
1738         boolean phyLocAlreadyExists =
1739                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1740                 dpnTepIp);
1741         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1742                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1743         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1744
1745         if (phyLocAlreadyExists) {
1746             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1747             return;
1748         }
1749         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1750         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1751     }
1752
1753     /**
1754      * Validate external tunnel state event.
1755      *
1756      * @param externalTunnel
1757      *            the external tunnel
1758      * @param intrf
1759      *            the intrf
1760      * @return true, if successful
1761      */
1762     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1763         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1764             String srcDevice = externalTunnel.getDestinationDevice();
1765             String destDevice = externalTunnel.getSourceDevice();
1766             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1767                     LogicalDatastoreType.CONFIGURATION);
1768             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1769                     otherEndPointExtTunnel);
1770             if (otherEndPointExtTunnel != null) {
1771                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1772                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1773                 if (otherEndPointInterfaceOperational) {
1774                     return true;
1775                 } else {
1776                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1777                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1778                 }
1779             }
1780         }
1781         return false;
1782     }
1783
1784     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1785         List<MatchInfo> mkMatches = new ArrayList<>();
1786         // Matching metadata
1787         mkMatches.add(
1788                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1789         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1790         return mkMatches;
1791     }
1792
1793     @Override
1794     protected ElanInterfaceManager getDataTreeChangeListener() {
1795         return this;
1796     }
1797
1798     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1799                                              BigInteger dpId) {
1800         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1801         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1802             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1803                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1804         try {
1805             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1806         } catch (InterruptedException e) {
1807             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1808         }
1809     }
1810 }