elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Lists;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.Queue;
  28 import java.util.Set;
  29 import java.util.concurrent.ConcurrentHashMap;
  30 import java.util.concurrent.ConcurrentLinkedQueue;
  31 import java.util.concurrent.ExecutionException;
  32 import java.util.concurrent.locks.ReentrantLock;
  33 import javax.annotation.Nullable;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 135 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 136 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 137 import org.slf4j.Logger;
 138 import org.slf4j.LoggerFactory;
 139
 140 /**
 141  * Class in charge of handling creations, modifications and removals of
 142  * ElanInterfaces.
 143  *
 144  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 145  */
 146 @Singleton
 147 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 148         implements RecoverableListener {
 149     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final boolean SH_FLAG_SET = true;
 152     private static final boolean SH_FLAG_UNSET = false;
 153
 154     private final DataBroker broker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalManager;
 157     private final IInterfaceManager interfaceManager;
 158     private final IdManagerService idManager;
 159     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 160     private final INeutronVpnManager neutronVpnManager;
 161     private final ElanItmUtils elanItmUtils;
 162     private final ElanEtreeUtils elanEtreeUtils;
 163     private final ElanL2GatewayUtils elanL2GatewayUtils;
 164     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 165     private final ElanUtils elanUtils;
 166     private final JobCoordinator jobCoordinator;
 167     private final ElanInstanceCache elanInstanceCache;
 168     private final ElanInterfaceCache elanInterfaceCache;
 169
 170     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 171         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 172
 173     @Inject
 174     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 175                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 176                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 177                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 178                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 179                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 180                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 181                                 final ElanInstanceCache elanInstanceCache,
 182                                 final ElanInterfaceCache elanInterfaceCache,
 183                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 184                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 185         super(ElanInterface.class, ElanInterfaceManager.class);
 186         this.broker = dataBroker;
 187         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 188         this.idManager = managerService;
 189         this.mdsalManager = mdsalApiManager;
 190         this.interfaceManager = interfaceManager;
 191         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 192         this.neutronVpnManager = neutronVpnManager;
 193         this.elanItmUtils = elanItmUtils;
 194         this.elanEtreeUtils = elanEtreeUtils;
 195         this.elanL2GatewayUtils = elanL2GatewayUtils;
 196         this.elanUtils = elanUtils;
 197         this.jobCoordinator = jobCoordinator;
 198         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 199         this.elanInstanceCache = elanInstanceCache;
 200         this.elanInterfaceCache = elanInterfaceCache;
 201         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 202     }
 203
 204     @Override
 205     @PostConstruct
 206     public void init() {
 207         registerListener();
 208     }
 209
 210     @Override
 211     public void registerListener() {
 212         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 213     }
 214
 215     @Override
 216     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 217         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 218     }
 219
 220     @Override
 221     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 222         String interfaceName = del.getName();
 223         String elanInstanceName = del.getElanInstanceName();
 224         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 225         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 226             elanInterfaces.remove(del);
 227             if (elanInterfaces.isEmpty()) {
 228                 unProcessedElanInterfaces.remove(elanInstanceName);
 229             }
 230         }
 231         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 232         /*
 233          * Handling in case the elan instance is deleted.If the Elan instance is
 234          * deleted, there is no need to explicitly delete the elan interfaces
 235          */
 236         if (elanInfo == null) {
 237             return;
 238         }
 239         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 240         if (interfaceInfo == null && elanInfo.isExternal()) {
 241             // In deleting external network, the underlying ietf Inteface might have been removed
 242             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 243             // from Operational DS instead
 244             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 245         }
 246         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 247                 interfaceName, interfaceInfo, this);
 248         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 249     }
 250
 251     private static class RemoveElanInterfaceHolder {
 252         boolean isLastElanInterface = false;
 253         boolean isLastInterfaceOnDpn = false;
 254         BigInteger dpId = null;
 255     }
 256
 257     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 258     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 259             InterfaceInfo interfaceInfo) {
 260         String elanName = elanInfo.getElanInstanceName();
 261         long elanTag = elanInfo.getElanTag();
 262         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 263         List<ListenableFuture<Void>> futures = new ArrayList<>();
 264         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 265         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 266             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 267             if (elanState == null) {
 268                 return;
 269             }
 270             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 271                 @Nullable List<String> elanInterfaces = elanState.getElanInterfaces();
 272                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 273                     holder.isLastElanInterface = true;
 274                 }
 275                 if (interfaceInfo != null) {
 276                     holder.dpId = interfaceInfo.getDpId();
 277                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 278                         interfaceName, elanTag, interfaceTx);
 279                     /*
 280                      * If there are not elan ports, remove the unknown dmac, terminating
 281                      * service table flows, remote/local bc group
 282                      */
 283                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 284                         || dpnInterfaces.getInterfaces().isEmpty()) {
 285                         // No more Elan Interfaces in this DPN
 286                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 287                             holder.dpId);
 288                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 289                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 290                         }
 291                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 292                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 293                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 294                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 296                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 297                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 298                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 299                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 300                             }
 301                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 302                         }
 303                         holder.isLastInterfaceOnDpn = true;
 304                     } else {
 305                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 306                     }
 307                 }
 308             }));
 309         }));
 310         futures.forEach(ElanUtils::waitForTransactionToComplete);
 311
 312         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 313             futures.add(
 314                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 315                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 316         }
 317         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 318                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 319         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 320                 ElanConstants.JOB_MAX_RETRIES);
 321
 322         return futures;
 323     }
 324
 325     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 326             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 327             throws ExecutionException, InterruptedException {
 328         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 329         if (etreeLeafTag != null) {
 330             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 331             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 332         }
 333     }
 334
 335     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 336             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 337             throws ExecutionException, InterruptedException {
 338         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 339         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340     }
 341
 342     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 346         if (etreeInstance != null) {
 347             BigInteger dpnId = interfaceInfo.getDpId();
 348             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 349             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 350             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 351         }
 352     }
 353
 354     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 358         if (etreeInstance != null) {
 359             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 360             BigInteger dpnId = interfaceInfo.getDpId();
 361             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 362             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 363             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 364         }
 365     }
 366
 367     private Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 368             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 369         String elanName = elanInfo.getElanInstanceName();
 370         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 371         if (elanState == null) {
 372             return elanState;
 373         }
 374         List<String> elanInterfaces = elanState.getElanInterfaces();
 375         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 376         if (!isRemoved) {
 377             return elanState;
 378         }
 379
 380         if (elanInterfaces.isEmpty()) {
 381             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 382             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 384         } else {
 385             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 386                     .withKey(new ElanKey(elanName)).build();
 387             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 388         }
 389         return elanState;
 390     }
 391
 392     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 393             throws ReadFailedException {
 394         // removing the ElanInterface from the config data_store if interface is
 395         // not present in Interface config DS
 396         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 397             interfaceName) == null
 398                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 399             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 400         }
 401     }
 402
 403     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 404             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 405         String elanName = elanInfo.getElanInstanceName();
 406         List<ListenableFuture<Void>> futures = new ArrayList<>();
 407         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 408             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 409                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 410                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 411                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 412                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 413                 if (interfaceInfo != null) {
 414                     if (existingElanInterfaceMac.isPresent()) {
 415                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 416                         if (existingMacEntries != null) {
 417                             List<PhysAddress> macAddresses = new ArrayList<>();
 418                             for (MacEntry macEntry : existingMacEntries) {
 419                                 PhysAddress macAddress = macEntry.getMacAddress();
 420                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 421                                         macAddress.getValue(), interfaceName);
 422                                 Optional<MacEntry> macEntryOptional =
 423                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 424                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 425                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 426                                 }
 427                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 428                                 macAddresses.add(macAddress);
 429                             }
 430
 431                             // Removing all those MACs from External Devices belonging
 432                             // to this ELAN
 433                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 434                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 435                             }
 436                         }
 437                     }
 438                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 439                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 440                 } else if (existingElanInterfaceMac.isPresent()) {
 441                     // Interface does not exist in ConfigDS, so lets remove everything
 442                     // about that interface related to Elan
 443                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 444                     if (macEntries != null) {
 445                         for (MacEntry macEntry : macEntries) {
 446                             PhysAddress macAddress = macEntry.getMacAddress();
 447                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 448                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 449                             }
 450                         }
 451                     }
 452                 }
 453                 if (existingElanInterfaceMac.isPresent()) {
 454                     interfaceTx.delete(elanInterfaceId);
 455                 }
 456                 unbindService(interfaceName, flowTx);
 457                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 458             }));
 459         }));
 460         return futures;
 461     }
 462
 463     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 464                                                                          String interfaceName, long elanTag,
 465                                                                          TypedReadWriteTransaction<Operational> tx)
 466             throws ExecutionException, InterruptedException {
 467         // FIXME: pass in and use ElanInstanceKey instead?
 468         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 469         lock.lock();
 470         try {
 471             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 472             if (dpnInterfaces != null) {
 473                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 474                 if (interfaceLists != null) {
 475                     interfaceLists.remove(interfaceName);
 476                 }
 477
 478                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 479                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 480                     deleteElanDpnInterface(elanName, dpId, tx);
 481                 } else {
 482                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 483                 }
 484             }
 485             return dpnInterfaces;
 486         } finally {
 487             lock.unlock();
 488         }
 489     }
 490
 491     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 492         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 493         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 494             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 495                 BigInteger currentDpId = dpnInterface.getDpId();
 496                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 497                     for (String elanInterface : dpnInterface.getInterfaces()) {
 498                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 499                         if (macs == null || macs.getMacEntry() == null) {
 500                             continue;
 501                         }
 502                         for (MacEntry mac : macs.getMacEntry()) {
 503                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 504                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 505                         }
 506                     }
 507                 }
 508             }
 509         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 510     }
 511
 512     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 513             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 514         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 515         if (etreeLeafTag != null) {
 516             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 517         }
 518     }
 519
 520     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 521             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 522         mdsalManager
 523                 .removeFlow(confTx, dpId,
 524                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 525                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 526                                         mac.getMacAddress().getValue(), elanTag)));
 527     }
 528
 529     /*
 530     * Possible Scenarios for update
 531     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 532         then all {1,2,3,4} should be removed
 533
 534         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 535         then all {1,2,3,4} should be added
 536
 537         c. if orig = {1,2,3,4} updated={2,3,4}
 538         then 1 should be removed
 539
 540         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 541         then we should just add 5
 542
 543         e. if orig = {1,2,3,4} updated={2,3,4,5}
 544         then 1 should be removed , 5 should be added
 545     * */
 546     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 547     @Override
 548     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 549         // updating the static-Mac Entries for the existing elanInterface
 550         String elanName = update.getElanInstanceName();
 551         String interfaceName = update.getName();
 552
 553         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 554         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 555         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 556         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 557
 558         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 559                 deletedEntry.getMacAddress()));
 560
 561         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 562         * Hence add the macentries for the same.*/
 563         for (StaticMacEntries staticMacEntry : updatedEntries) {
 564             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 565                     staticMacEntry.getMacAddress());
 566             ListenableFutures.addErrorLogging(ElanUtils.waitForTransactionToComplete(
 567                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 568                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 569                     if (existingMacEntry.isPresent()) {
 570                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 571                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 572                             tx);
 573                     } else {
 574                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 575                             elanName, interfaceName, staticMacEntry, tx);
 576                     }
 577                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 578         }
 579     }
 580
 581     @Override
 582     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 583         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 584             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 585             String interfaceName = elanInterfaceAdded.getName();
 586             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 587             if (interfaceInfo == null) {
 588                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 589                 return;
 590             }
 591             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 592
 593             if (elanInstance == null) {
 594                 // Add the ElanInstance in the Configuration data-store
 595                 List<String> elanInterfaces = new ArrayList<>();
 596                 elanInterfaces.add(interfaceName);
 597                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 598                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 599                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 600                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 601             }
 602
 603             Long elanTag = elanInstance.getElanTag();
 604             // If elan tag is not updated, then put the elan interface into
 605             // unprocessed entry map and entry. Let entries
 606             // in this map get processed during ELAN update DCN.
 607             if (elanTag == null || elanTag == 0L) {
 608                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 609                 if (elanInterfaces == null) {
 610                     elanInterfaces = new ConcurrentLinkedQueue<>();
 611                 }
 612                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 613                     elanInterfaces.add(elanInterfaceAdded);
 614                 }
 615                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 616                 return;
 617             }
 618             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 619                     interfaceInfo, elanInstance, this);
 620             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 621         }), LOG, "Error processing added ELAN interface");
 622     }
 623
 624     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 625         List<ListenableFuture<Void>> futures = new ArrayList<>();
 626         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 627         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 628             return futures;
 629         }
 630         for (ElanInterface elanInterface : elanInterfaces) {
 631             String interfaceName = elanInterface.getName();
 632             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 633             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 634         }
 635         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 636         return futures;
 637     }
 638
 639     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 640             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 641         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 642                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 643         List<DpnInterfaces> dpnInterfaceLists = null;
 644         if (elanDpnInterfacesList != null) {
 645             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 646         }
 647         if (dpnInterfaceLists == null) {
 648             dpnInterfaceLists = new ArrayList<>();
 649         }
 650         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 651             BigInteger dstDpId = interfaceInfo.getDpId();
 652             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 653                 continue;
 654             }
 655             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 656             for (String remoteIf : remoteElanInterfaces) {
 657                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 658                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 659                 if (elanIfMac == null || remoteInterface == null) {
 660                     continue;
 661                 }
 662                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 663                 if (remoteMacEntries != null) {
 664                     for (MacEntry macEntry : remoteMacEntries) {
 665                         String macAddress = macEntry.getMacAddress().getValue();
 666                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 667                                 dstDpId, elanInstance.getElanInstanceName());
 668                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 669                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 670                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 671                     }
 672                 }
 673             }
 674         }
 675     }
 676
 677     private static class AddElanInterfaceHolder {
 678         private DpnInterfaces dpnInterfaces = null;
 679         private boolean isFirstInterfaceInDpn = false;
 680         private BigInteger dpId;
 681     }
 682
 683     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 684     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 685             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 686         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 687         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 688         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 689
 690         String interfaceName = elanInterface.getName();
 691         String elanInstanceName = elanInterface.getElanInstanceName();
 692
 693         List<ListenableFuture<Void>> futures = new ArrayList<>();
 694         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 695         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 696             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 697             if (elanInfo == null) {
 698                 List<String> elanInterfaces = new ArrayList<>();
 699                 elanInterfaces.add(interfaceName);
 700                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 701                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 702                         operTx)));
 703             } else {
 704                 createElanStateList(elanInstanceName, interfaceName, operTx);
 705             }
 706             // Specific actions to the DPN where the ElanInterface has been added,
 707             // for example, programming the
 708             // External tunnel table if needed or adding the ElanInterface to the
 709             // DpnInterfaces in the operational DS.
 710             holder.dpId = interfaceInfo.getDpId();
 711             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 712                 // FIXME: use elanInstaince.key() instead?
 713                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 714                 lock.lock();
 715                 try {
 716                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 717                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 718                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 719                     if (ElanUtils.isVlan(elanInstance)) {
 720                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 721                             elanInstanceName, existingElanDpnInterfaces);
 722                     } else {
 723                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 724                     }
 725                     if (holder.isFirstInterfaceInDpn) {
 726                         // ELAN's 1st ElanInterface added to this DPN
 727                         if (!existingElanDpnInterfaces.isPresent()) {
 728                             holder.dpnInterfaces =
 729                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 730                         } else {
 731                             @Nullable List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 732                             List<String> elanInterfaces =
 733                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 734                             elanInterfaces.add(interfaceName);
 735                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 736                                 elanInterfaces, operTx);
 737                         }
 738                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 739                         // table, but only if Elan has VNI
 740                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 741                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 742                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 743                         }
 744                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 745                             interfaceName);
 746                     } else {
 747                         @Nullable List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 748                         List<String> elanInterfaces =
 749                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 750                         elanInterfaces.add(interfaceName);
 751                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 752                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 753                                 interfaceName);
 754                         }
 755                         holder.dpnInterfaces =
 756                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 757                     }
 758                 } finally {
 759                     lock.unlock();
 760                 }
 761             }
 762
 763             // add code to install Local/Remote BC group, unknow DMAC entry,
 764             // terminating service table flow entry
 765             // call bindservice of interfacemanager to create ingress table flow
 766             // enty.
 767             // Add interface to the ElanInterfaceForwardingEntires Container
 768             createElanInterfaceTablesList(interfaceName, operTx);
 769         }));
 770         futures.forEach(ElanUtils::waitForTransactionToComplete);
 771         futures.add(
 772             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 773                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 774                     holder.isFirstInterfaceInDpn, confTx))));
 775
 776         // add the vlan provider interface to remote BC group for the elan
 777         // for internal vlan networks
 778         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 779             if (interfaceManager.isExternalInterface(interfaceName)) {
 780                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 781                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 782             }
 783         }
 784
 785         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 786             //update the remote-DPNs remoteBC group entry with Tunnels
 787             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 788                 holder.dpId);
 789             futures.add(
 790                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 791                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 792         }
 793
 794         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 795         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 796                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 797         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 798         return futures;
 799     }
 800
 801     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 802     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 803             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 804         String elanInstanceName = elanInstance.getElanInstanceName();
 805         String interfaceName = elanInterface.getName();
 806         List<ListenableFuture<Void>> futures = new ArrayList<>();
 807         BigInteger dpId = interfaceInfo.getDpId();
 808         boolean isInterfaceOperational = isOperational(interfaceInfo);
 809         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 810             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 811                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 812                     isFirstInterfaceInDpn, confTx);
 813
 814                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 815                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 816
 817                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 818                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 819                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 820                             staticMacEntry.getMacAddress());
 821                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 822                             LogicalDatastoreType.OPERATIONAL, macId);
 823                         if (existingMacEntry.isPresent()) {
 824                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 825                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 826                                 existingMacEntry.get(), operTx);
 827                         } else {
 828                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 829                                 interfaceName, staticMacEntry, operTx);
 830                         }
 831
 832                         if (isInterfaceOperational) {
 833                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 834                             // DPNs
 835                             String macAddress = staticMacEntry.getMacAddress().getValue();
 836                             LOG.info(
 837                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 838                                     + " {}",
 839                                 macAddress, elanInstanceName, interfaceName);
 840                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 841                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 842                         }
 843                     }
 844
 845                     if (isInterfaceOperational) {
 846                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 847                         // on purpose.
 848                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 849                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 850                         }
 851                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 852                             staticMacAddresses);
 853                     }
 854                 }
 855             }));
 856         }));
 857         futures.forEach(ElanUtils::waitForTransactionToComplete);
 858         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 859             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 860             try {
 861                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 862                 if (neutronPort != null) {
 863                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 864                 }
 865             } catch (IllegalArgumentException ex) {
 866                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 867             }
 868         }
 869         return futures;
 870     }
 871
 872     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 873             PhysAddress physAddress) {
 874         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 875         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 876         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 877                 LogicalDatastoreType.OPERATIONAL, macId);
 878
 879         if (!existingMacEntry.isPresent()) {
 880             return;
 881         }
 882
 883         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 884                 .withKey(new MacEntryKey(physAddress)).build();
 885         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 886                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 887     }
 888
 889     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 890             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 891         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 892         if (!existingElanDpnInterfaces.isPresent()) {
 893             return true;
 894         }
 895         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 896             return false;
 897         }
 898         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 899         int dummyInterfaceCount =  0;
 900         List<String> interfaces = dpnInterfaces.getInterfaces();
 901         if (interfaces == null) {
 902             return true;
 903         }
 904         if (interfaces.contains(routerPortUuid)) {
 905             dummyInterfaceCount++;
 906         }
 907         if (interfaces.contains(elanInstanceName)) {
 908             dummyInterfaceCount++;
 909         }
 910         return interfaces.size() == dummyInterfaceCount;
 911     }
 912
 913     private InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName, PhysAddress physAddress) {
 914         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 915                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 916     }
 917
 918     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 919             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 920         if (!isOperational(interfaceInfo)) {
 921             return;
 922         }
 923         BigInteger dpId = interfaceInfo.getDpId();
 924         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 925             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 926         }
 927         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 928         if (isFirstInterfaceInDpn) {
 929             // Terminating Service , UnknownDMAC Table.
 930             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 931             // for ELAN Instance is VxLAN
 932             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 933                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 934             }
 935             setupUnknownDMacTable(elanInstance, dpId, confTx);
 936             /*
 937              * Install remote DMAC flow. This is required since this DPN is
 938              * added later to the elan instance and remote DMACs of other
 939              * interfaces in this elan instance are not present in the current
 940              * dpn.
 941              */
 942             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 943                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 944                         elanInstance.getElanInstanceName());
 945                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 946             }
 947         }
 948         // bind the Elan service to the Interface
 949         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 950     }
 951
 952     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 953             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 954         if (!isOperational(interfaceInfo)) {
 955             return;
 956         }
 957         // LocalBroadcast Group creation with elan-Interfaces
 958         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 959         if (isFirstInterfaceInDpn) {
 960             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 961             BigInteger dpId = interfaceInfo.getDpId();
 962             // RemoteBroadcast Group creation
 963             try {
 964                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 965             } catch (InterruptedException e1) {
 966                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 967             }
 968             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 969             try {
 970                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 971             } catch (InterruptedException e1) {
 972                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 973             }
 974         }
 975     }
 976
 977     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 978             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 979         int ifTag = interfaceInfo.getInterfaceTag();
 980         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 981                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 982                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 983                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 984                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 985
 986         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 987
 988         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 989                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 990                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 991                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 992
 993         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 994     }
 995
 996     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 997             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
 998         int ifTag = interfaceInfo.getInterfaceTag();
 999         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1000             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1001
1002         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1003
1004         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1005             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1006             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1007             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1008
1009         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1010     }
1011
1012     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1013             TypedWriteTransaction<Configuration> confTx) {
1014         int elanTag = elanInfo.getElanTag().intValue();
1015         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1016         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1017         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1018         if (etreeInstance != null) {
1019             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1020             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1021             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1022         }
1023     }
1024
1025     @SuppressWarnings("checkstyle:IllegalCatch")
1026     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1027             TypedWriteTransaction<Configuration> confTx) {
1028         int bucketId = 0;
1029         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1030         if (elanDpns != null) {
1031             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1032             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1033                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1034                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1035                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1036                     List<Action> listAction = new ArrayList<>();
1037                     int actionKey = 0;
1038                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1039                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1040                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1041                     bucketId++;
1042                     for (DpnInterfaces otherFes : dpnInterfaces) {
1043                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1044                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1045                             && !otherFes.getInterfaces().isEmpty()) {
1046                             try {
1047                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1048                                         dpnInterface.getDpId(), otherFes.getDpId(),
1049                                         elanUtils.isOpenstackVniSemanticsEnforced()
1050                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1051                                 if (!remoteListActionInfo.isEmpty()) {
1052                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1053                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1054                                     bucketId++;
1055                                 }
1056                             } catch (Exception ex) {
1057                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1058                                         + "Logical Group Interface not found between source Dpn - {}, "
1059                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1060                                 return;
1061                             }
1062                         }
1063                     }
1064                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1065                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1066                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1067
1068                     if (remoteListBucketInfo.isEmpty()) {
1069                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1070                         continue;
1071                     }
1072                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1073                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1074                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1075                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1076                 }
1077             }
1078             try {
1079                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1080             } catch (InterruptedException e1) {
1081                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1082             }
1083         }
1084     }
1085
1086     private List<MatchInfo> buildMatchesForVni(Long vni) {
1087         List<MatchInfo> mkMatches = new ArrayList<>();
1088         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1089         mkMatches.add(match);
1090         return mkMatches;
1091     }
1092
1093     private List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1094         List<InstructionInfo> mkInstructions = new ArrayList<>();
1095         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1096         return mkInstructions;
1097     }
1098
1099     private List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1100         List<MatchInfo> mkMatches = new ArrayList<>();
1101         // Matching metadata
1102         mkMatches.add(new MatchMetadata(
1103                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1104         return mkMatches;
1105     }
1106
1107     /**
1108      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1109      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1110      *
1111      * @param elanTag
1112      *            elanTag to be written in metadata when flow is selected
1113      * @return the instructions ready to be installed in a flow
1114      */
1115     private List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1116         List<InstructionInfo> mkInstructions = new ArrayList<>();
1117         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1118                 .getElanMetadataMask()));
1119         /* applicable for EXTERNAL_TUNNEL_TABLE only
1120         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1121         */
1122         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1123         return mkInstructions;
1124     }
1125
1126     // Install DMAC entry on dst DPN
1127     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1128             BigInteger dstDpId) {
1129         String interfaceName = interfaceInfo.getInterfaceName();
1130         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1131         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1132             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1133             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1134                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1135                     for (MacEntry macEntry : macEntries) {
1136                         String macAddress = macEntry.getMacAddress().getValue();
1137                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1138                             interfaceName);
1139                         synchronized (ElanUtils.getElanMacDPNKey(elanInfo.getElanTag(), macAddress,
1140                             interfaceInfo.getDpId())) {
1141                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1142                                 macAddress);
1143                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1144                         }
1145                     }
1146                 })));
1147         }
1148         return emptyList();
1149     }
1150
1151     private void createDropBucket(List<Bucket> listBucket) {
1152         List<Action> actionsInfos = new ArrayList<>();
1153         actionsInfos.add(new ActionDrop().buildAction());
1154         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1155                 MDSALUtil.WATCH_GROUP);
1156         listBucket.add(dropBucket);
1157     }
1158
1159     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1160             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1161         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1162         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1163     }
1164
1165     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1166             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1167         List<Bucket> listBucket = new ArrayList<>();
1168         int bucketId = 0;
1169         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1170
1171         List<String> interfaces = new ArrayList<>();
1172         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1173             interfaces = newDpnInterface.getInterfaces();
1174         }
1175         for (String ifName : interfaces) {
1176             // In case if there is a InterfacePort in the cache which is not in
1177             // operational state, skip processing it
1178             InterfaceInfo ifInfo = interfaceManager
1179                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1180             if (!isOperational(ifInfo)) {
1181                 continue;
1182             }
1183
1184             if (!interfaceManager.isExternalInterface(ifName)) {
1185                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1186                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1187                 bucketId++;
1188             }
1189         }
1190
1191         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1192                 MDSALUtil.buildBucketLists(listBucket));
1193         LOG.trace("installing the localBroadCast Group:{}", group);
1194         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1195     }
1196
1197     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1198             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1199         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1200         if (etreeInstance != null) {
1201             List<Bucket> listBucket = new ArrayList<>();
1202             int bucketId = 0;
1203
1204             List<String> interfaces = new ArrayList<>();
1205             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1206                 interfaces = newDpnInterface.getInterfaces();
1207             }
1208             for (String ifName : interfaces) {
1209                 // In case if there is a InterfacePort in the cache which is not
1210                 // in
1211                 // operational state, skip processing it
1212                 InterfaceInfo ifInfo = interfaceManager
1213                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1214                 if (!isOperational(ifInfo)) {
1215                     continue;
1216                 }
1217
1218                 if (!interfaceManager.isExternalInterface(ifName)) {
1219                     // only add root interfaces
1220                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1221                 }
1222             }
1223
1224             if (listBucket.isEmpty()) { // No Buckets
1225                 createDropBucket(listBucket);
1226             }
1227
1228             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1229             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1230             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1231                     MDSALUtil.buildBucketLists(listBucket));
1232             LOG.trace("installing the localBroadCast Group:{}", group);
1233             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1234         }
1235     }
1236
1237     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1238             InterfaceInfo ifInfo) {
1239         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1240         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1241             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1242                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1243             bucketId++;
1244         }
1245         return bucketId;
1246     }
1247
1248     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1249             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1250             throws ExecutionException, InterruptedException {
1251         BigInteger dpnId = interfaceInfo.getDpId();
1252         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1253         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1254         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1255     }
1256
1257     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1258             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1259             throws ExecutionException, InterruptedException {
1260         BigInteger dpnId = interfaceInfo.getDpId();
1261         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1262         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1263         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1264     }
1265
1266     /**
1267      * Installs a flow in the External Tunnel table consisting in translating
1268      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1269      * elanTag that will be used later in the ELANs pipeline.
1270      * @param dpnId the dpn id
1271      */
1272     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1273             TypedWriteTransaction<Configuration> confTx) {
1274         long elanTag = elanInfo.getElanTag();
1275         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1276                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1277                 elanInfo.getElanInstanceName(), // flowName
1278                 0, // idleTimeout
1279                 0, // hardTimeout
1280                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1281                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1282                 getInstructionsIntOrExtTunnelTable(elanTag));
1283
1284         mdsalManager.addFlow(confTx, flowEntity);
1285     }
1286
1287     /**
1288      * Removes, from External Tunnel table, the flow that translates from VNI to
1289      * elanTag. Important: ensure this method is only called whenever there is
1290      * no other ElanInterface in the specified DPN
1291      *
1292      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1293      */
1294     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1295             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1296         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1297         // last ElanInstance plus
1298         // adding a new one does (almost at the same time) are executed in that
1299         // exact order
1300
1301         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1302         FlowEntity flowEntity = new FlowEntityBuilder()
1303             .setDpnId(dpnId)
1304             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1305             .setFlowId(flowId)
1306             .build();
1307         mdsalManager.removeFlow(confTx, flowEntity);
1308     }
1309
1310     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1311             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1312         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1313         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1314     }
1315
1316     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1317             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1318         List<? extends MatchInfoBase> listMatchInfoBase;
1319         List<InstructionInfo> instructionInfos;
1320         long serviceId;
1321         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1322             serviceId = elanTag;
1323             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1324             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1325         } else {
1326             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1327             listMatchInfoBase = buildMatchesForVni(serviceId);
1328             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1329         }
1330         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1331                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1332                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1333                 instructionInfos);
1334         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1335     }
1336
1337     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1338             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1339         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1340         if (etreeInstance != null) {
1341             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1342         }
1343     }
1344
1345     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1346             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1347         long elanTag = elanInfo.getElanTag();
1348         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1349         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1350         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1351     }
1352
1353     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1354             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1355         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1356         if (etreeLeafTag != null) {
1357             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1358             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1359             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1360         }
1361     }
1362
1363     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1364             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1365         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1366                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1367                 5, elanInfo.getElanInstanceName(), 0, 0,
1368                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1369                 getMatchesForElanTag(elanTag, /* SH flag */false),
1370                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1371
1372         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1373     }
1374
1375     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1376             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1377         // only if ELAN can connect to external network, perform the following
1378
1379         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1380             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1381                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1382                     5, elanInfo.getElanInstanceName(), 0, 0,
1383                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1384                     getMatchesForElanTag(elanTag, /* SH flag */true),
1385                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1386             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1387         }
1388     }
1389
1390
1391     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1392             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1393             throws ExecutionException, InterruptedException {
1394         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1395                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1396         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1397
1398         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1399             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1400                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1401                     .build();
1402             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1403         }
1404     }
1405
1406     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1407         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1408     }
1409
1410     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1411             TypedWriteTransaction<Configuration> tx) {
1412         if (isStandardElanService(elanInterface)) {
1413             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1414                     elanInterface.getName(), lportTag, tx);
1415         } else { // Etree service
1416             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1417         }
1418     }
1419
1420     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1421             TypedWriteTransaction<Configuration> tx) {
1422         int instructionKey = 0;
1423         List<Instruction> instructions = new ArrayList<>();
1424         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1425                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1426
1427         List<Action> actions = new ArrayList<>();
1428         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1429                 lportTag).buildAction());
1430         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1431                 elanTag).buildAction());
1432         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1433
1434         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1435                 ++instructionKey));
1436
1437         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1438         BoundServices serviceInfo = ElanUtils.getBoundServices(
1439                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1440                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1441         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1442         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1443                 bindServiceId);
1444         if (!existingElanService.isPresent()) {
1445             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1446         }
1447     }
1448
1449     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1450             TypedWriteTransaction<Configuration> tx) {
1451         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1452             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1453                     lportTag, tx);
1454         } else {
1455             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1456             if (etreeInstance == null) {
1457                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1458                         elanInterface.getName(), elanInfo.getElanInstanceName());
1459             } else {
1460                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1461                         elanInterface.getName(), lportTag, tx);
1462             }
1463         }
1464     }
1465
1466     private boolean isStandardElanService(ElanInterface elanInterface) {
1467         return elanInterface.augmentation(EtreeInterface.class) == null;
1468     }
1469
1470     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1471             throws ExecutionException, InterruptedException {
1472         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1473         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1474         if (tx.read(bindServiceId).get().isPresent()) {
1475             tx.delete(bindServiceId);
1476         }
1477     }
1478
1479     private String getFlowRef(long tableId, long elanTag) {
1480         return String.valueOf(tableId) + elanTag;
1481     }
1482
1483     private String getFlowRef(long tableId, long elanTag, String flowName) {
1484         return new StringBuffer().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1485                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1486     }
1487
1488     private String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1489         return String.valueOf(tableId) + elanTag + shFlag;
1490     }
1491
1492     private List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1493         List<Action> listAction = new ArrayList<>();
1494         int actionKey = 0;
1495         listAction.add(
1496             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1497         actionKey++;
1498         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1499         return listAction;
1500     }
1501
1502     private DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1503             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1504         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1505                 .withKey(new DpnInterfacesKey(dpId)).build();
1506         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1507                 CREATE_MISSING_PARENTS);
1508         return dpnInterface;
1509     }
1510
1511     /**
1512      * Delete elan dpn interface from operational DS.
1513      *
1514      * @param elanInstanceName
1515      *            the elan instance name
1516      * @param dpId
1517      *            the dp id
1518      */
1519     private void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1520             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1521         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1522                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1523         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1524         if (dpnInterfaces.isPresent()) {
1525             tx.delete(dpnInterfacesId);
1526         }
1527     }
1528
1529     private DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName, BigInteger dpId,
1530             TypedWriteTransaction<Operational> tx) {
1531         List<String> interfaceNames = new ArrayList<>();
1532         interfaceNames.add(interfaceName);
1533         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1534                 .withKey(new DpnInterfacesKey(dpId)).build();
1535         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1536                 CREATE_MISSING_PARENTS);
1537         return dpnInterface;
1538     }
1539
1540     private void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1541             throws ExecutionException, InterruptedException {
1542         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1543                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1544         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1545         // Adding new Elan Interface Port to the operational DataStore without
1546         // Static-Mac Entries..
1547         if (!interfaceMacTables.isPresent()) {
1548             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1549                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1550             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1551                     CREATE_MISSING_PARENTS);
1552         }
1553     }
1554
1555     private void createElanStateList(String elanInstanceName, String interfaceName,
1556             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1557         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1558         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1559         // Adding new Elan Interface Port to the operational DataStore without
1560         // Static-Mac Entries..
1561         if (elanInterfaceLists.isPresent()) {
1562             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1563             if (interfaceLists == null) {
1564                 interfaceLists = new ArrayList<>();
1565             }
1566             interfaceLists.add(interfaceName);
1567             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1568                     .withKey(new ElanKey(elanInstanceName)).build();
1569             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1570         }
1571     }
1572
1573     private boolean isOperational(InterfaceInfo interfaceInfo) {
1574         if (interfaceInfo == null) {
1575             return false;
1576         }
1577         return interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1578     }
1579
1580     @SuppressWarnings("checkstyle:IllegalCatch")
1581     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1582         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1583         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1584                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1585         if (dpnInterfaceLists == null) {
1586             return;
1587         }
1588         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1589         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1590             int cnt = 0;
1591             String elanName = elanDpns.getElanInstanceName();
1592             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1593             if (elanInfo == null) {
1594                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1595                         + "skipping this ELAN for tunnel handling", elanName);
1596                 continue;
1597             }
1598             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1599                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1600                 continue;
1601             }
1602             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1603             if (dpnInterfaces == null) {
1604                 continue;
1605             }
1606             DpnInterfaces dstDpnIf = null;
1607             for (DpnInterfaces dpnIf : dpnInterfaces) {
1608                 BigInteger dpnIfDpId = dpnIf.getDpId();
1609                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1610                     cnt++;
1611                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1612                     cnt++;
1613                     dstDpnIf = dpnIf;
1614                 }
1615             }
1616             if (cnt == 2) {
1617                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1618                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1619                 jobCoordinator.enqueueJob(elanName, () -> {
1620                     // update Remote BC Group
1621                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1622                     try {
1623                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1624                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1625                                 confTx)).get();
1626                     } catch (RuntimeException e) {
1627                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1628                     }
1629                     Set<String> interfaceLists = new HashSet<>();
1630                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1631                     for (String ifName : interfaceLists) {
1632                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1633                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1634                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1635                             if (isOperational(interfaceInfo)) {
1636                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1637                             }
1638                             return emptyList();
1639                         }, ElanConstants.JOB_MAX_RETRIES);
1640                     }
1641                     return emptyList();
1642                 }, ElanConstants.JOB_MAX_RETRIES);
1643             }
1644
1645         }
1646     }
1647
1648     /**
1649      * Handle external tunnel state event.
1650      *
1651      * @param externalTunnel
1652      *            the external tunnel
1653      * @param intrf
1654      *            the interface
1655      */
1656     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1657         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1658             return;
1659         }
1660         // dpId/externalNodeId will be available either in source or destination
1661         // based on the tunnel end point
1662         BigInteger dpId = null;
1663         NodeId externalNodeId = null;
1664         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1665             dpId = new BigInteger(externalTunnel.getSourceDevice());
1666             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1667         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1668             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1669             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1670         }
1671         if (dpId == null || externalNodeId == null) {
1672             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1673             return;
1674         }
1675
1676         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1677         if (dpnInterfaceLists == null) {
1678             return;
1679         }
1680         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1681         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1682             String elanName = elanDpns.getElanInstanceName();
1683             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1684
1685             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1686             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1687                     || dpnInterfaces.getInterfaces().isEmpty()) {
1688                 continue;
1689             }
1690             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1691
1692             final BigInteger finalDpId = dpId;
1693             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1694                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1695                 "Error setting up ELAN BGs");
1696             // install L2gwDevices local macs in dpn.
1697             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1698             // Install dpn macs on external device
1699             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1700                     externalNodeId);
1701         }
1702         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1703     }
1704
1705     /**
1706      * Installs dpn macs in external device. first it checks if the physical
1707      * locator towards this dpn tep is present or not if the physical locator is
1708      * present go ahead and add the ucast macs otherwise update the mcast mac
1709      * entry to include this dpn tep ip and schedule the job to put ucast macs
1710      * once the physical locator is programmed in device
1711      *
1712      * @param elanName
1713      *            the elan name
1714      * @param lstElanInterfaceNames
1715      *            the lst Elan interface names
1716      * @param dpnId
1717      *            the dpn id
1718      * @param externalNodeId
1719      *            the external node id
1720      */
1721     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1722             NodeId externalNodeId) {
1723         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1724                 externalNodeId.getValue());
1725         if (elanL2GwDevice == null) {
1726             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1727             return;
1728         }
1729         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1730         if (dpnTepIp == null) {
1731             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1732                     externalNodeId);
1733             return;
1734         }
1735
1736         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1737         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1738                 LogicalDatastoreType.OPERATIONAL);
1739         boolean phyLocAlreadyExists =
1740                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1741                 dpnTepIp);
1742         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1743                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1744         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1745
1746         if (phyLocAlreadyExists) {
1747             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1748             return;
1749         }
1750         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1751         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1752     }
1753
1754     /**
1755      * Validate external tunnel state event.
1756      *
1757      * @param externalTunnel
1758      *            the external tunnel
1759      * @param intrf
1760      *            the intrf
1761      * @return true, if successful
1762      */
1763     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1764         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1765             String srcDevice = externalTunnel.getDestinationDevice();
1766             String destDevice = externalTunnel.getSourceDevice();
1767             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1768                     LogicalDatastoreType.CONFIGURATION);
1769             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1770                     otherEndPointExtTunnel);
1771             if (otherEndPointExtTunnel != null) {
1772                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1773                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1774                 if (otherEndPointInterfaceOperational) {
1775                     return true;
1776                 } else {
1777                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1778                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1779                 }
1780             }
1781         }
1782         return false;
1783     }
1784
1785     private List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1786         List<MatchInfo> mkMatches = new ArrayList<>();
1787         // Matching metadata
1788         mkMatches.add(
1789                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1790         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1791         return mkMatches;
1792     }
1793
1794     @Override
1795     protected ElanInterfaceManager getDataTreeChangeListener() {
1796         return this;
1797     }
1798
1799     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1800                                              BigInteger dpId) {
1801         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1802         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1803             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1804                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1805         try {
1806             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1807         } catch (InterruptedException e) {
1808             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1809         }
1810     }
1811 }