elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 136 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 138 import org.slf4j.Logger;
 139 import org.slf4j.LoggerFactory;
 140
 141 /**
 142  * Class in charge of handling creations, modifications and removals of
 143  * ElanInterfaces.
 144  *
 145  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 146  */
 147 @Singleton
 148 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 149         implements RecoverableListener {
 150     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 151     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 152     private static final boolean SH_FLAG_SET = true;
 153     private static final boolean SH_FLAG_UNSET = false;
 154
 155     private final DataBroker broker;
 156     private final ManagedNewTransactionRunner txRunner;
 157     private final IMdsalApiManager mdsalManager;
 158     private final IInterfaceManager interfaceManager;
 159     private final IdManagerService idManager;
 160     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 161     private final INeutronVpnManager neutronVpnManager;
 162     private final ElanItmUtils elanItmUtils;
 163     private final ElanEtreeUtils elanEtreeUtils;
 164     private final ElanL2GatewayUtils elanL2GatewayUtils;
 165     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 166     private final ElanUtils elanUtils;
 167     private final JobCoordinator jobCoordinator;
 168     private final ElanInstanceCache elanInstanceCache;
 169     private final ElanInterfaceCache elanInterfaceCache;
 170
 171     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 172         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 173
 174     @Inject
 175     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 176                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 177                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 178                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 179                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 180                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 181                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 182                                 final ElanInstanceCache elanInstanceCache,
 183                                 final ElanInterfaceCache elanInterfaceCache,
 184                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 185                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 186         super(ElanInterface.class, ElanInterfaceManager.class);
 187         this.broker = dataBroker;
 188         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 189         this.idManager = managerService;
 190         this.mdsalManager = mdsalApiManager;
 191         this.interfaceManager = interfaceManager;
 192         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 193         this.neutronVpnManager = neutronVpnManager;
 194         this.elanItmUtils = elanItmUtils;
 195         this.elanEtreeUtils = elanEtreeUtils;
 196         this.elanL2GatewayUtils = elanL2GatewayUtils;
 197         this.elanUtils = elanUtils;
 198         this.jobCoordinator = jobCoordinator;
 199         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 200         this.elanInstanceCache = elanInstanceCache;
 201         this.elanInterfaceCache = elanInterfaceCache;
 202         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 203     }
 204
 205     @Override
 206     @PostConstruct
 207     public void init() {
 208         registerListener();
 209     }
 210
 211     @Override
 212     public void registerListener() {
 213         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 214     }
 215
 216     @Override
 217     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 218         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 219     }
 220
 221     @Override
 222     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 223         String interfaceName = del.getName();
 224         String elanInstanceName = del.getElanInstanceName();
 225         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 226         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 227             elanInterfaces.remove(del);
 228             if (elanInterfaces.isEmpty()) {
 229                 unProcessedElanInterfaces.remove(elanInstanceName);
 230             }
 231         }
 232         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 233         /*
 234          * Handling in case the elan instance is deleted.If the Elan instance is
 235          * deleted, there is no need to explicitly delete the elan interfaces
 236          */
 237         if (elanInfo == null) {
 238             return;
 239         }
 240         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 241         if (interfaceInfo == null && elanInfo.isExternal()) {
 242             // In deleting external network, the underlying ietf Inteface might have been removed
 243             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 244             // from Operational DS instead
 245             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 246         }
 247         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 248                 interfaceName, interfaceInfo, this);
 249         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 250     }
 251
 252     private static class RemoveElanInterfaceHolder {
 253         boolean isLastElanInterface = false;
 254         boolean isLastInterfaceOnDpn = false;
 255         BigInteger dpId = null;
 256     }
 257
 258     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 259     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 260             InterfaceInfo interfaceInfo) {
 261         String elanName = elanInfo.getElanInstanceName();
 262         long elanTag = elanInfo.getElanTag();
 263         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 264         List<ListenableFuture<Void>> futures = new ArrayList<>();
 265         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 266         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 267             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 268             if (elanState == null) {
 269                 return;
 270             }
 271             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 272                 List<String> elanInterfaces = elanState.getElanInterfaces();
 273                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 274                     holder.isLastElanInterface = true;
 275                 }
 276                 if (interfaceInfo != null) {
 277                     holder.dpId = interfaceInfo.getDpId();
 278                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 279                         interfaceName, elanTag, interfaceTx);
 280                     /*
 281                      * If there are not elan ports, remove the unknown dmac, terminating
 282                      * service table flows, remote/local bc group
 283                      */
 284                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 285                         || dpnInterfaces.getInterfaces().isEmpty()) {
 286                         // No more Elan Interfaces in this DPN
 287                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 288                             holder.dpId);
 289                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 290                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag());
 291                         }
 292                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag());
 293                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 294                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 295                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 296                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 297                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 298                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 299                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 300                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 301                             }
 302                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 303                         }
 304                         holder.isLastInterfaceOnDpn = true;
 305                     } else {
 306                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 307                     }
 308                 }
 309             }));
 310         }));
 311         futures.forEach(ElanUtils::waitForTransactionToComplete);
 312
 313         if (holder.isLastInterfaceOnDpn && holder.dpId != null && isVxlanNetworkOrVxlanSegment(elanInfo)) {
 314             futures.add(
 315                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 316                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInfo, holder.dpId, confTx))));
 317         }
 318         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 319                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 320         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 321                 ElanConstants.JOB_MAX_RETRIES);
 322
 323         return futures;
 324     }
 325
 326     private void removeEtreeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
 327             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 328             throws ExecutionException, InterruptedException {
 329         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag());
 330         if (etreeLeafTag != null) {
 331             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
 332             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 333         }
 334     }
 335
 336     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 337             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 338             throws ExecutionException, InterruptedException {
 339         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 340         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 341     }
 342
 343     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 344             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 345             throws ExecutionException, InterruptedException {
 346         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 347         if (etreeInstance != null) {
 348             BigInteger dpnId = interfaceInfo.getDpId();
 349             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue());
 350             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 351             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 352         }
 353     }
 354
 355     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 356             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 357             throws ExecutionException, InterruptedException {
 358         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 359         if (etreeInstance != null) {
 360             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue();
 361             BigInteger dpnId = interfaceInfo.getDpId();
 362             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 363             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 364             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 365         }
 366     }
 367
 368     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 369             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 370         String elanName = elanInfo.getElanInstanceName();
 371         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 372         if (elanState == null) {
 373             return elanState;
 374         }
 375         List<String> elanInterfaces = elanState.getElanInterfaces();
 376         boolean isRemoved = elanInterfaces != null && elanInterfaces.remove(interfaceName);
 377         if (!isRemoved) {
 378             return elanState;
 379         }
 380
 381         if (elanInterfaces.isEmpty()) {
 382             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 383             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 384             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 385             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 386         } else {
 387             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 388                     .withKey(new ElanKey(elanName)).build();
 389             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 390         }
 391         return elanState;
 392     }
 393
 394     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 395             throws ReadFailedException {
 396         // removing the ElanInterface from the config data_store if interface is
 397         // not present in Interface config DS
 398         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 399             interfaceName) == null
 400                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 401             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 402         }
 403     }
 404
 405     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 406             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 407         String elanName = elanInfo.getElanInstanceName();
 408         List<ListenableFuture<Void>> futures = new ArrayList<>();
 409         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 410             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 411                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 412                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 413                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 414                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 415                 if (interfaceInfo != null) {
 416                     if (existingElanInterfaceMac.isPresent()) {
 417                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 418                         if (existingMacEntries != null) {
 419                             List<PhysAddress> macAddresses = new ArrayList<>();
 420                             for (MacEntry macEntry : existingMacEntries) {
 421                                 PhysAddress macAddress = macEntry.getMacAddress();
 422                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 423                                         macAddress.getValue(), interfaceName);
 424                                 Optional<MacEntry> macEntryOptional =
 425                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 426                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 427                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 428                                 }
 429                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 430                                 macAddresses.add(macAddress);
 431                             }
 432
 433                             // Removing all those MACs from External Devices belonging
 434                             // to this ELAN
 435                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 436                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 437                             }
 438                         }
 439                     }
 440                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 441                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 442                 } else if (existingElanInterfaceMac.isPresent()) {
 443                     // Interface does not exist in ConfigDS, so lets remove everything
 444                     // about that interface related to Elan
 445                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 446                     if (macEntries != null) {
 447                         for (MacEntry macEntry : macEntries) {
 448                             PhysAddress macAddress = macEntry.getMacAddress();
 449                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 450                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 451                             }
 452                         }
 453                     }
 454                 }
 455                 if (existingElanInterfaceMac.isPresent()) {
 456                     interfaceTx.delete(elanInterfaceId);
 457                 }
 458                 unbindService(interfaceName, flowTx);
 459                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 460             }));
 461         }));
 462         return futures;
 463     }
 464
 465     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, BigInteger dpId,
 466                                                                          String interfaceName, long elanTag,
 467                                                                          TypedReadWriteTransaction<Operational> tx)
 468             throws ExecutionException, InterruptedException {
 469         // FIXME: pass in and use ElanInstanceKey instead?
 470         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 471         lock.lock();
 472         try {
 473             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 474             if (dpnInterfaces != null) {
 475                 List<String> interfaceLists = dpnInterfaces.getInterfaces();
 476                 if (interfaceLists != null) {
 477                     interfaceLists.remove(interfaceName);
 478                 }
 479
 480                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 481                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 482                     deleteElanDpnInterface(elanName, dpId, tx);
 483                 } else {
 484                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 485                 }
 486             }
 487             return dpnInterfaces;
 488         } finally {
 489             lock.unlock();
 490         }
 491     }
 492
 493     private void deleteAllRemoteMacsInADpn(String elanName, BigInteger dpId, long elanTag) {
 494         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 495         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 496             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 497                 BigInteger currentDpId = dpnInterface.getDpId();
 498                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 499                     for (String elanInterface : dpnInterface.getInterfaces()) {
 500                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 501                         if (macs == null || macs.getMacEntry() == null) {
 502                             continue;
 503                         }
 504                         for (MacEntry mac : macs.getMacEntry()) {
 505                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 506                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 507                         }
 508                     }
 509                 }
 510             }
 511         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 512     }
 513
 514     private void removeEtreeMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 515             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 516         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
 517         if (etreeLeafTag != null) {
 518             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), currentDpId, mac, confTx);
 519         }
 520     }
 521
 522     private void removeTheMacFlowInTheDPN(BigInteger dpId, long elanTag, BigInteger currentDpId, MacEntry mac,
 523             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 524         mdsalManager
 525                 .removeFlow(confTx, dpId,
 526                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 527                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 528                                         mac.getMacAddress().getValue(), elanTag)));
 529     }
 530
 531     /*
 532     * Possible Scenarios for update
 533     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 534         then all {1,2,3,4} should be removed
 535
 536         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 537         then all {1,2,3,4} should be added
 538
 539         c. if orig = {1,2,3,4} updated={2,3,4}
 540         then 1 should be removed
 541
 542         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 543         then we should just add 5
 544
 545         e. if orig = {1,2,3,4} updated={2,3,4,5}
 546         then 1 should be removed , 5 should be added
 547     * */
 548     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 549     @Override
 550     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 551         // updating the static-Mac Entries for the existing elanInterface
 552         String elanName = update.getElanInstanceName();
 553         String interfaceName = update.getName();
 554
 555         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 556         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 557         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 558         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 559
 560         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 561                 deletedEntry.getMacAddress()));
 562
 563         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 564         * Hence add the macentries for the same.*/
 565         for (StaticMacEntries staticMacEntry : updatedEntries) {
 566             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 567                     staticMacEntry.getMacAddress());
 568             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 569                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 570                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 571                     if (existingMacEntry.isPresent()) {
 572                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 573                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 574                             tx);
 575                     } else {
 576                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 577                             elanName, interfaceName, staticMacEntry, tx);
 578                     }
 579                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 580         }
 581     }
 582
 583     @Override
 584     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 585         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 586             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 587             String interfaceName = elanInterfaceAdded.getName();
 588             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 589             if (interfaceInfo == null) {
 590                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 591                 return;
 592             }
 593             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 594
 595             if (elanInstance == null) {
 596                 // Add the ElanInstance in the Configuration data-store
 597                 List<String> elanInterfaces = new ArrayList<>();
 598                 elanInterfaces.add(interfaceName);
 599                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 600                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 601                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 602                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 603             }
 604
 605             Long elanTag = elanInstance.getElanTag();
 606             // If elan tag is not updated, then put the elan interface into
 607             // unprocessed entry map and entry. Let entries
 608             // in this map get processed during ELAN update DCN.
 609             if (elanTag == null || elanTag == 0L) {
 610                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 611                 if (elanInterfaces == null) {
 612                     elanInterfaces = new ConcurrentLinkedQueue<>();
 613                 }
 614                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 615                     elanInterfaces.add(elanInterfaceAdded);
 616                 }
 617                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 618                 return;
 619             }
 620             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 621                     interfaceInfo, elanInstance, this);
 622             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 623         }), LOG, "Error processing added ELAN interface");
 624     }
 625
 626     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 627         List<ListenableFuture<Void>> futures = new ArrayList<>();
 628         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 629         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 630             return futures;
 631         }
 632         for (ElanInterface elanInterface : elanInterfaces) {
 633             String interfaceName = elanInterface.getName();
 634             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 635             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 636         }
 637         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 638         return futures;
 639     }
 640
 641     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 642             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 643         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 644                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 645         List<DpnInterfaces> dpnInterfaceLists = null;
 646         if (elanDpnInterfacesList != null) {
 647             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 648         }
 649         if (dpnInterfaceLists == null) {
 650             dpnInterfaceLists = new ArrayList<>();
 651         }
 652         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 653             BigInteger dstDpId = interfaceInfo.getDpId();
 654             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 655                 continue;
 656             }
 657             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 658             for (String remoteIf : remoteElanInterfaces) {
 659                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 660                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 661                 if (elanIfMac == null || remoteInterface == null) {
 662                     continue;
 663                 }
 664                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 665                 if (remoteMacEntries != null) {
 666                     for (MacEntry macEntry : remoteMacEntries) {
 667                         String macAddress = macEntry.getMacAddress().getValue();
 668                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 669                                 dstDpId, elanInstance.getElanInstanceName());
 670                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 671                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 672                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 673                     }
 674                 }
 675             }
 676         }
 677     }
 678
 679     private static class AddElanInterfaceHolder {
 680         private DpnInterfaces dpnInterfaces = null;
 681         private boolean isFirstInterfaceInDpn = false;
 682         private BigInteger dpId;
 683     }
 684
 685     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 686     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 687             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 688         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 689         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 690         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 691
 692         String interfaceName = elanInterface.getName();
 693         String elanInstanceName = elanInterface.getElanInstanceName();
 694
 695         List<ListenableFuture<Void>> futures = new ArrayList<>();
 696         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 697         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 698             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 699             if (elanInfo == null) {
 700                 List<String> elanInterfaces = new ArrayList<>();
 701                 elanInterfaces.add(interfaceName);
 702                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 703                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 704                         operTx)));
 705             } else {
 706                 createElanStateList(elanInstanceName, interfaceName, operTx);
 707             }
 708             // Specific actions to the DPN where the ElanInterface has been added,
 709             // for example, programming the
 710             // External tunnel table if needed or adding the ElanInterface to the
 711             // DpnInterfaces in the operational DS.
 712             holder.dpId = interfaceInfo.getDpId();
 713             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 714                 // FIXME: use elanInstaince.key() instead?
 715                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 716                 lock.lock();
 717                 try {
 718                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 719                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 720                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 721                     if (ElanUtils.isVlan(elanInstance)) {
 722                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 723                             elanInstanceName, existingElanDpnInterfaces);
 724                     } else {
 725                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 726                     }
 727                     if (holder.isFirstInterfaceInDpn) {
 728                         // ELAN's 1st ElanInterface added to this DPN
 729                         if (!existingElanDpnInterfaces.isPresent()) {
 730                             holder.dpnInterfaces =
 731                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 732                         } else {
 733                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 734                             List<String> elanInterfaces =
 735                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 736                             elanInterfaces.add(interfaceName);
 737                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 738                                 elanInterfaces, operTx);
 739                         }
 740                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 741                         // table, but only if Elan has VNI
 742                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 743                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 744                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 745                         }
 746                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 747                             interfaceName);
 748                     } else {
 749                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 750                         List<String> elanInterfaces =
 751                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 752                         elanInterfaces.add(interfaceName);
 753                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 754                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 755                                 interfaceName);
 756                         }
 757                         holder.dpnInterfaces =
 758                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 759                     }
 760                 } finally {
 761                     lock.unlock();
 762                 }
 763             }
 764
 765             // add code to install Local/Remote BC group, unknow DMAC entry,
 766             // terminating service table flow entry
 767             // call bindservice of interfacemanager to create ingress table flow
 768             // enty.
 769             // Add interface to the ElanInterfaceForwardingEntires Container
 770             createElanInterfaceTablesList(interfaceName, operTx);
 771         }));
 772         futures.forEach(ElanUtils::waitForTransactionToComplete);
 773         futures.add(
 774             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 775                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 776                     holder.isFirstInterfaceInDpn, confTx))));
 777
 778         // add the vlan provider interface to remote BC group for the elan
 779         // for internal vlan networks
 780         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 781             if (interfaceManager.isExternalInterface(interfaceName)) {
 782                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 783                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 784             }
 785         }
 786
 787         if (holder.isFirstInterfaceInDpn && isVxlanNetworkOrVxlanSegment(elanInstance)) {
 788             //update the remote-DPNs remoteBC group entry with Tunnels
 789             LOG.trace("update remote bc group for elan {} on other DPNs for newly added dpn {}", elanInstance,
 790                 holder.dpId);
 791             futures.add(
 792                 ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 793                     confTx -> setElanAndEtreeBCGrouponOtherDpns(elanInstance, holder.dpId, confTx))));
 794         }
 795
 796         String jobKey = ElanUtils.getElanInterfaceJobKey(interfaceName);
 797         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(jobKey,
 798                 elanInterface, interfaceInfo, elanInstance, holder.isFirstInterfaceInDpn, this);
 799         jobCoordinator.enqueueJob(jobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
 800         return futures;
 801     }
 802
 803     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 804     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 805             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 806         String elanInstanceName = elanInstance.getElanInstanceName();
 807         String interfaceName = elanInterface.getName();
 808         List<ListenableFuture<Void>> futures = new ArrayList<>();
 809         BigInteger dpId = interfaceInfo.getDpId();
 810         boolean isInterfaceOperational = isOperational(interfaceInfo);
 811         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 812             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 813                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 814                     isFirstInterfaceInDpn, confTx);
 815
 816                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 817                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 818
 819                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 820                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 821                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 822                             staticMacEntry.getMacAddress());
 823                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 824                             LogicalDatastoreType.OPERATIONAL, macId);
 825                         if (existingMacEntry.isPresent()) {
 826                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 827                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 828                                 existingMacEntry.get(), operTx);
 829                         } else {
 830                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 831                                 interfaceName, staticMacEntry, operTx);
 832                         }
 833
 834                         if (isInterfaceOperational) {
 835                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 836                             // DPNs
 837                             String macAddress = staticMacEntry.getMacAddress().getValue();
 838                             LOG.info(
 839                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 840                                     + " {}",
 841                                 macAddress, elanInstanceName, interfaceName);
 842                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 843                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 844                         }
 845                     }
 846
 847                     if (isInterfaceOperational) {
 848                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 849                         // on purpose.
 850                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 851                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 852                         }
 853                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 854                             staticMacAddresses);
 855                     }
 856                 }
 857             }));
 858         }));
 859         futures.forEach(ElanUtils::waitForTransactionToComplete);
 860         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 861             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 862             try {
 863                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 864                 if (neutronPort != null) {
 865                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 866                 }
 867             } catch (IllegalArgumentException ex) {
 868                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 869             }
 870         }
 871         return futures;
 872     }
 873
 874     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 875             PhysAddress physAddress) {
 876         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 877         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 878         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 879                 LogicalDatastoreType.OPERATIONAL, macId);
 880
 881         if (!existingMacEntry.isPresent()) {
 882             return;
 883         }
 884
 885         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 886                 .withKey(new MacEntryKey(physAddress)).build();
 887         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 888                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 889     }
 890
 891     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 892             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 893         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 894         if (!existingElanDpnInterfaces.isPresent()) {
 895             return true;
 896         }
 897         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 898             return false;
 899         }
 900         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 901         int dummyInterfaceCount =  0;
 902         List<String> interfaces = dpnInterfaces.getInterfaces();
 903         if (interfaces == null) {
 904             return true;
 905         }
 906         if (interfaces.contains(routerPortUuid)) {
 907             dummyInterfaceCount++;
 908         }
 909         if (interfaces.contains(elanInstanceName)) {
 910             dummyInterfaceCount++;
 911         }
 912         return interfaces.size() == dummyInterfaceCount;
 913     }
 914
 915     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 916             PhysAddress physAddress) {
 917         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 918                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 919     }
 920
 921     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 922             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 923         if (!isOperational(interfaceInfo)) {
 924             return;
 925         }
 926         BigInteger dpId = interfaceInfo.getDpId();
 927         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 928             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 929         }
 930         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 931         if (isFirstInterfaceInDpn) {
 932             // Terminating Service , UnknownDMAC Table.
 933             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 934             // for ELAN Instance is VxLAN
 935             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 936                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 937             }
 938             setupUnknownDMacTable(elanInstance, dpId, confTx);
 939             /*
 940              * Install remote DMAC flow. This is required since this DPN is
 941              * added later to the elan instance and remote DMACs of other
 942              * interfaces in this elan instance are not present in the current
 943              * dpn.
 944              */
 945             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 946                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 947                         elanInstance.getElanInstanceName());
 948                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 949             }
 950         }
 951         // bind the Elan service to the Interface
 952         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 953     }
 954
 955     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 956             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 957         if (!isOperational(interfaceInfo)) {
 958             return;
 959         }
 960         // LocalBroadcast Group creation with elan-Interfaces
 961         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
 962         if (isFirstInterfaceInDpn) {
 963             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
 964             BigInteger dpId = interfaceInfo.getDpId();
 965             // RemoteBroadcast Group creation
 966             try {
 967                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 968             } catch (InterruptedException e1) {
 969                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 970             }
 971             elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, dpnInterfaces, dpId, confTx);
 972             try {
 973                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 974             } catch (InterruptedException e1) {
 975                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
 976             }
 977         }
 978     }
 979
 980     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 981             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 982         int ifTag = interfaceInfo.getInterfaceTag();
 983         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 984                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
 985                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 986                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
 987                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
 988
 989         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
 990
 991         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
 992                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
 993                 0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
 994                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
 995
 996         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
 997     }
 998
 999     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1000             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1001         int ifTag = interfaceInfo.getInterfaceTag();
1002         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1003             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1004
1005         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1006
1007         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1008             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1009             0, ElanConstants.COOKIE_ELAN_FILTER_EQUALS.add(BigInteger.valueOf(ifTag)),
1010             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1011
1012         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1013     }
1014
1015     private void setElanAndEtreeBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId,
1016             TypedWriteTransaction<Configuration> confTx) {
1017         int elanTag = elanInfo.getElanTag().intValue();
1018         long groupId = ElanUtils.getElanRemoteBCGId(elanTag);
1019         setBCGrouponOtherDpns(elanInfo, dpId, elanTag, groupId, confTx);
1020         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1021         if (etreeInstance != null) {
1022             int etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().intValue();
1023             long etreeLeafGroupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeLeafTag);
1024             setBCGrouponOtherDpns(elanInfo, dpId, etreeLeafTag, etreeLeafGroupId, confTx);
1025         }
1026     }
1027
1028     @SuppressWarnings("checkstyle:IllegalCatch")
1029     private void setBCGrouponOtherDpns(ElanInstance elanInfo, BigInteger dpId, int elanTag, long groupId,
1030             TypedWriteTransaction<Configuration> confTx) {
1031         int bucketId = 0;
1032         ElanDpnInterfacesList elanDpns = elanUtils.getElanDpnInterfacesList(elanInfo.getElanInstanceName());
1033         if (elanDpns != null) {
1034             List<DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1035             for (DpnInterfaces dpnInterface : dpnInterfaces) {
1036                 List<Bucket> remoteListBucketInfo = new ArrayList<>();
1037                 if (elanUtils.isDpnPresent(dpnInterface.getDpId()) && !Objects.equals(dpnInterface.getDpId(), dpId)
1038                         && dpnInterface.getInterfaces() != null && !dpnInterface.getInterfaces().isEmpty()) {
1039                     List<Action> listAction = new ArrayList<>();
1040                     int actionKey = 0;
1041                     listAction.add(new ActionGroup(ElanUtils.getElanLocalBCGId(elanTag)).buildAction(++actionKey));
1042                     remoteListBucketInfo.add(MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, bucketId,
1043                             MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1044                     bucketId++;
1045                     for (DpnInterfaces otherFes : dpnInterfaces) {
1046                         if (elanUtils.isDpnPresent(otherFes.getDpId()) && !Objects.equals(otherFes.getDpId(),
1047                             dpnInterface.getDpId()) && otherFes.getInterfaces() != null
1048                             && !otherFes.getInterfaces().isEmpty()) {
1049                             try {
1050                                 List<Action> remoteListActionInfo = elanItmUtils.getInternalTunnelItmEgressAction(
1051                                         dpnInterface.getDpId(), otherFes.getDpId(),
1052                                         elanUtils.isOpenstackVniSemanticsEnforced()
1053                                                 ? ElanUtils.getVxlanSegmentationId(elanInfo) : elanTag);
1054                                 if (!remoteListActionInfo.isEmpty()) {
1055                                     remoteListBucketInfo.add(MDSALUtil.buildBucket(remoteListActionInfo, MDSALUtil
1056                                             .GROUP_WEIGHT, bucketId, MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1057                                     bucketId++;
1058                                 }
1059                             } catch (Exception ex) {
1060                                 LOG.error("setElanBCGrouponOtherDpns failed due to Exception caught; "
1061                                         + "Logical Group Interface not found between source Dpn - {}, "
1062                                         + "destination Dpn - {} ", dpnInterface.getDpId(), otherFes.getDpId(), ex);
1063                                 return;
1064                             }
1065                         }
1066                     }
1067                     List<Bucket> elanL2GwDevicesBuckets = elanL2GatewayMulticastUtils
1068                             .getRemoteBCGroupBucketsOfElanL2GwDevices(elanInfo, dpnInterface.getDpId(), bucketId);
1069                     remoteListBucketInfo.addAll(elanL2GwDevicesBuckets);
1070
1071                     if (remoteListBucketInfo.isEmpty()) {
1072                         LOG.debug("No ITM is present on Dpn - {} ", dpnInterface.getDpId());
1073                         continue;
1074                     }
1075                     Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1076                             MDSALUtil.buildBucketLists(remoteListBucketInfo));
1077                     LOG.trace("Installing remote bc group {} on dpnId {}", group, dpnInterface.getDpId());
1078                     mdsalManager.addGroup(confTx, dpnInterface.getDpId(), group);
1079                 }
1080             }
1081             try {
1082                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1083             } catch (InterruptedException e1) {
1084                 LOG.warn("Error while waiting for remote BC group on other DPNs for ELAN {} to install", elanInfo);
1085             }
1086         }
1087     }
1088
1089     private static List<MatchInfo> buildMatchesForVni(Long vni) {
1090         List<MatchInfo> mkMatches = new ArrayList<>();
1091         MatchInfo match = new MatchTunnelId(BigInteger.valueOf(vni));
1092         mkMatches.add(match);
1093         return mkMatches;
1094     }
1095
1096     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1097         List<InstructionInfo> mkInstructions = new ArrayList<>();
1098         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1099         return mkInstructions;
1100     }
1101
1102     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1103         List<MatchInfo> mkMatches = new ArrayList<>();
1104         // Matching metadata
1105         mkMatches.add(new MatchMetadata(
1106                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1107         return mkMatches;
1108     }
1109
1110     /**
1111      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1112      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1113      *
1114      * @param elanTag
1115      *            elanTag to be written in metadata when flow is selected
1116      * @return the instructions ready to be installed in a flow
1117      */
1118     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Long elanTag) {
1119         List<InstructionInfo> mkInstructions = new ArrayList<>();
1120         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag), ElanHelper
1121                 .getElanMetadataMask()));
1122         /* applicable for EXTERNAL_TUNNEL_TABLE only
1123         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1124         */
1125         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1126         return mkInstructions;
1127     }
1128
1129     // Install DMAC entry on dst DPN
1130     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1131             BigInteger dstDpId) {
1132         String interfaceName = interfaceInfo.getInterfaceName();
1133         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1134         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1135             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1136             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1137                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1138                     for (MacEntry macEntry : macEntries) {
1139                         String macAddress = macEntry.getMacAddress().getValue();
1140                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1141                             interfaceName);
1142                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag(), macAddress,
1143                             interfaceInfo.getDpId())) {
1144                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1145                                 macAddress);
1146                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1147                         }
1148                     }
1149                 })));
1150         }
1151         return emptyList();
1152     }
1153
1154     private static void createDropBucket(List<Bucket> listBucket) {
1155         List<Action> actionsInfos = new ArrayList<>();
1156         actionsInfos.add(new ActionDrop().buildAction());
1157         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1158                 MDSALUtil.WATCH_GROUP);
1159         listBucket.add(dropBucket);
1160     }
1161
1162     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1163             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1164         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1165         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1166     }
1167
1168     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1169             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1170         List<Bucket> listBucket = new ArrayList<>();
1171         int bucketId = 0;
1172         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1173
1174         List<String> interfaces = new ArrayList<>();
1175         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1176             interfaces = newDpnInterface.getInterfaces();
1177         }
1178         for (String ifName : interfaces) {
1179             // In case if there is a InterfacePort in the cache which is not in
1180             // operational state, skip processing it
1181             InterfaceInfo ifInfo = interfaceManager
1182                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1183             if (!isOperational(ifInfo)) {
1184                 continue;
1185             }
1186
1187             if (!interfaceManager.isExternalInterface(ifName)) {
1188                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1189                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1190                 bucketId++;
1191             }
1192         }
1193
1194         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1195                 MDSALUtil.buildBucketLists(listBucket));
1196         LOG.trace("installing the localBroadCast Group:{}", group);
1197         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1198     }
1199
1200     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1201             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1202         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1203         if (etreeInstance != null) {
1204             List<Bucket> listBucket = new ArrayList<>();
1205             int bucketId = 0;
1206
1207             List<String> interfaces = new ArrayList<>();
1208             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1209                 interfaces = newDpnInterface.getInterfaces();
1210             }
1211             for (String ifName : interfaces) {
1212                 // In case if there is a InterfacePort in the cache which is not
1213                 // in
1214                 // operational state, skip processing it
1215                 InterfaceInfo ifInfo = interfaceManager
1216                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1217                 if (!isOperational(ifInfo)) {
1218                     continue;
1219                 }
1220
1221                 if (!interfaceManager.isExternalInterface(ifName)) {
1222                     // only add root interfaces
1223                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1224                 }
1225             }
1226
1227             if (listBucket.isEmpty()) { // No Buckets
1228                 createDropBucket(listBucket);
1229             }
1230
1231             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue();
1232             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1233             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1234                     MDSALUtil.buildBucketLists(listBucket));
1235             LOG.trace("installing the localBroadCast Group:{}", group);
1236             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1237         }
1238     }
1239
1240     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1241             InterfaceInfo ifInfo) {
1242         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1243         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1244             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1245                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1246             bucketId++;
1247         }
1248         return bucketId;
1249     }
1250
1251     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1252             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1253             throws ExecutionException, InterruptedException {
1254         BigInteger dpnId = interfaceInfo.getDpId();
1255         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag());
1256         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1257         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1258     }
1259
1260     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1261             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1262             throws ExecutionException, InterruptedException {
1263         BigInteger dpnId = interfaceInfo.getDpId();
1264         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag());
1265         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1266         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1267     }
1268
1269     /**
1270      * Installs a flow in the External Tunnel table consisting in translating
1271      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1272      * elanTag that will be used later in the ELANs pipeline.
1273      * @param dpnId the dpn id
1274      */
1275     private void setExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1276             TypedWriteTransaction<Configuration> confTx) {
1277         long elanTag = elanInfo.getElanTag();
1278         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1279                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag), 5, // prio
1280                 elanInfo.getElanInstanceName(), // flowName
1281                 0, // idleTimeout
1282                 0, // hardTimeout
1283                 ITMConstants.COOKIE_ITM_EXTERNAL.add(BigInteger.valueOf(elanTag)),
1284                 buildMatchesForVni(ElanUtils.getVxlanSegmentationId(elanInfo)),
1285                 getInstructionsIntOrExtTunnelTable(elanTag));
1286
1287         mdsalManager.addFlow(confTx, flowEntity);
1288     }
1289
1290     /**
1291      * Removes, from External Tunnel table, the flow that translates from VNI to
1292      * elanTag. Important: ensure this method is only called whenever there is
1293      * no other ElanInterface in the specified DPN
1294      *
1295      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1296      */
1297     private void unsetExternalTunnelTable(BigInteger dpnId, ElanInstance elanInfo,
1298             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1299         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1300         // last ElanInstance plus
1301         // adding a new one does (almost at the same time) are executed in that
1302         // exact order
1303
1304         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag());
1305         FlowEntity flowEntity = new FlowEntityBuilder()
1306             .setDpnId(dpnId)
1307             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1308             .setFlowId(flowId)
1309             .build();
1310         mdsalManager.removeFlow(confTx, flowEntity);
1311     }
1312
1313     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1314             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1315         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1316         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1317     }
1318
1319     public void setupTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1320             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1321         List<? extends MatchInfoBase> listMatchInfoBase;
1322         List<InstructionInfo> instructionInfos;
1323         long serviceId;
1324         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1325             serviceId = elanTag;
1326             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId((int) elanTag);
1327             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag));
1328         } else {
1329             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1330             listMatchInfoBase = buildMatchesForVni(serviceId);
1331             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1332         }
1333         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1334                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId), 5, String.format("%s:%d", "ITM Flow Entry ",
1335                 elanTag), 0, 0, ITMConstants.COOKIE_ITM.add(BigInteger.valueOf(elanTag)), listMatchInfoBase,
1336                 instructionInfos);
1337         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1338     }
1339
1340     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, BigInteger dpId,
1341             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1342         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1343         if (etreeInstance != null) {
1344             setupTerminateServiceTable(elanInfo, dpId, etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1345         }
1346     }
1347
1348     public void setupUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId,
1349             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1350         long elanTag = elanInfo.getElanTag();
1351         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1352         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1353         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1354     }
1355
1356     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1357             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1358         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1359         if (etreeLeafTag != null) {
1360             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue();
1361             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1362             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1363         }
1364     }
1365
1366     private void installLocalUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1367             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1368         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1369                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1370                 5, elanInfo.getElanInstanceName(), 0, 0,
1371                 ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1372                 getMatchesForElanTag(elanTag, /* SH flag */false),
1373                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1374
1375         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1376     }
1377
1378     private void installRemoteUnknownFlow(ElanInstance elanInfo, BigInteger dpId, long elanTag,
1379             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1380         // only if ELAN can connect to external network, perform the following
1381
1382         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1383             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1384                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1385                     5, elanInfo.getElanInstanceName(), 0, 0,
1386                     ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.add(BigInteger.valueOf(elanTag)),
1387                     getMatchesForElanTag(elanTag, /* SH flag */true),
1388                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1389             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1390         }
1391     }
1392
1393
1394     private void removeUnknownDmacFlow(BigInteger dpId, ElanInstance elanInfo,
1395             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1396             throws ExecutionException, InterruptedException {
1397         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1398                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1399         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1400
1401         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1402             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1403                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1404                     .build();
1405             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1406         }
1407     }
1408
1409     private void removeDefaultTermFlow(BigInteger dpId, long elanTag) {
1410         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1411     }
1412
1413     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1414             TypedWriteTransaction<Configuration> tx) {
1415         if (isStandardElanService(elanInterface)) {
1416             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(),
1417                     elanInterface.getName(), lportTag, tx);
1418         } else { // Etree service
1419             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1420         }
1421     }
1422
1423     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1424             TypedWriteTransaction<Configuration> tx) {
1425         int instructionKey = 0;
1426         List<Instruction> instructions = new ArrayList<>();
1427         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1428                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1429
1430         List<Action> actions = new ArrayList<>();
1431         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1432                 lportTag).buildAction());
1433         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1434                 elanTag).buildAction());
1435         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1436
1437         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1438                 ++instructionKey));
1439
1440         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1441         BoundServices serviceInfo = ElanUtils.getBoundServices(
1442                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1443                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1444         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1445         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1446                 bindServiceId);
1447         if (!existingElanService.isPresent()) {
1448             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1449         }
1450     }
1451
1452     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1453             TypedWriteTransaction<Configuration> tx) {
1454         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1455             bindElanService(elanInfo.getElanTag(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1456                     lportTag, tx);
1457         } else {
1458             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1459             if (etreeInstance == null) {
1460                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1461                         elanInterface.getName(), elanInfo.getElanInstanceName());
1462             } else {
1463                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue(), elanInfo.getElanInstanceName(),
1464                         elanInterface.getName(), lportTag, tx);
1465             }
1466         }
1467     }
1468
1469     private static boolean isStandardElanService(ElanInterface elanInterface) {
1470         return elanInterface.augmentation(EtreeInterface.class) == null;
1471     }
1472
1473     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1474             throws ExecutionException, InterruptedException {
1475         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1476         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1477         if (tx.read(bindServiceId).get().isPresent()) {
1478             tx.delete(bindServiceId);
1479         }
1480     }
1481
1482     private static String getFlowRef(long tableId, long elanTag) {
1483         return String.valueOf(tableId) + elanTag;
1484     }
1485
1486     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1487         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1488                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1489     }
1490
1491     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1492         return String.valueOf(tableId) + elanTag + shFlag;
1493     }
1494
1495     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1496         List<Action> listAction = new ArrayList<>();
1497         int actionKey = 0;
1498         listAction.add(
1499             new ActionSetFieldTunnelId(BigInteger.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1500         actionKey++;
1501         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1502         return listAction;
1503     }
1504
1505     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, BigInteger dpId,
1506             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1507         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1508                 .withKey(new DpnInterfacesKey(dpId)).build();
1509         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1510                 CREATE_MISSING_PARENTS);
1511         return dpnInterface;
1512     }
1513
1514     /**
1515      * Delete elan dpn interface from operational DS.
1516      *
1517      * @param elanInstanceName
1518      *            the elan instance name
1519      * @param dpId
1520      *            the dp id
1521      */
1522     private static void deleteElanDpnInterface(String elanInstanceName, BigInteger dpId,
1523             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1524         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1525                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1526         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1527         if (dpnInterfaces.isPresent()) {
1528             tx.delete(dpnInterfacesId);
1529         }
1530     }
1531
1532     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1533             BigInteger dpId, TypedWriteTransaction<Operational> tx) {
1534         List<String> interfaceNames = new ArrayList<>();
1535         interfaceNames.add(interfaceName);
1536         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1537                 .withKey(new DpnInterfacesKey(dpId)).build();
1538         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1539                 CREATE_MISSING_PARENTS);
1540         return dpnInterface;
1541     }
1542
1543     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1544             throws ExecutionException, InterruptedException {
1545         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1546                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1547         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1548         // Adding new Elan Interface Port to the operational DataStore without
1549         // Static-Mac Entries..
1550         if (!interfaceMacTables.isPresent()) {
1551             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1552                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1553             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1554                     CREATE_MISSING_PARENTS);
1555         }
1556     }
1557
1558     private static void createElanStateList(String elanInstanceName, String interfaceName,
1559             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1560         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1561         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1562         // Adding new Elan Interface Port to the operational DataStore without
1563         // Static-Mac Entries..
1564         if (elanInterfaceLists.isPresent()) {
1565             List<String> interfaceLists = elanInterfaceLists.get().getElanInterfaces();
1566             if (interfaceLists == null) {
1567                 interfaceLists = new ArrayList<>();
1568             }
1569             interfaceLists.add(interfaceName);
1570             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1571                     .withKey(new ElanKey(elanInstanceName)).build();
1572             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1573         }
1574     }
1575
1576     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1577         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1578     }
1579
1580     @SuppressWarnings("checkstyle:IllegalCatch")
1581     public void handleInternalTunnelStateEvent(BigInteger srcDpId, BigInteger dstDpId) {
1582         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1583         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1584                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1585         if (dpnInterfaceLists == null) {
1586             return;
1587         }
1588         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1589         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1590             int cnt = 0;
1591             String elanName = elanDpns.getElanInstanceName();
1592             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1593             if (elanInfo == null) {
1594                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1595                         + "skipping this ELAN for tunnel handling", elanName);
1596                 continue;
1597             }
1598             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1599                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1600                 continue;
1601             }
1602             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1603             if (dpnInterfaces == null) {
1604                 continue;
1605             }
1606             DpnInterfaces dstDpnIf = null;
1607             for (DpnInterfaces dpnIf : dpnInterfaces) {
1608                 BigInteger dpnIfDpId = dpnIf.getDpId();
1609                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1610                     cnt++;
1611                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1612                     cnt++;
1613                     dstDpnIf = dpnIf;
1614                 }
1615             }
1616             if (cnt == 2) {
1617                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1618                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1619                 jobCoordinator.enqueueJob(elanName, () -> {
1620                     // update Remote BC Group
1621                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1622                     try {
1623                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1624                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1625                                 confTx)).get();
1626                     } catch (RuntimeException e) {
1627                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1628                     }
1629                     Set<String> interfaceLists = new HashSet<>();
1630                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1631                     for (String ifName : interfaceLists) {
1632                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1633                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1634                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1635                             if (isOperational(interfaceInfo)) {
1636                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1637                             }
1638                             return emptyList();
1639                         }, ElanConstants.JOB_MAX_RETRIES);
1640                     }
1641                     return emptyList();
1642                 }, ElanConstants.JOB_MAX_RETRIES);
1643             }
1644
1645         }
1646     }
1647
1648     /**
1649      * Handle external tunnel state event.
1650      *
1651      * @param externalTunnel
1652      *            the external tunnel
1653      * @param intrf
1654      *            the interface
1655      */
1656     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1657         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1658             return;
1659         }
1660         // dpId/externalNodeId will be available either in source or destination
1661         // based on the tunnel end point
1662         BigInteger dpId = null;
1663         NodeId externalNodeId = null;
1664         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1665             dpId = new BigInteger(externalTunnel.getSourceDevice());
1666             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1667         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1668             dpId = new BigInteger(externalTunnel.getDestinationDevice());
1669             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1670         }
1671         if (dpId == null || externalNodeId == null) {
1672             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1673             return;
1674         }
1675
1676         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1677         if (dpnInterfaceLists == null) {
1678             return;
1679         }
1680         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1681         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1682             String elanName = elanDpns.getElanInstanceName();
1683             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1684
1685             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1686             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1687                     || dpnInterfaces.getInterfaces().isEmpty()) {
1688                 continue;
1689             }
1690             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1691
1692             final BigInteger finalDpId = dpId;
1693             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1694                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1695                 "Error setting up ELAN BGs");
1696             // install L2gwDevices local macs in dpn.
1697             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1698             // Install dpn macs on external device
1699             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1700                     externalNodeId);
1701         }
1702         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1703     }
1704
1705     /**
1706      * Installs dpn macs in external device. first it checks if the physical
1707      * locator towards this dpn tep is present or not if the physical locator is
1708      * present go ahead and add the ucast macs otherwise update the mcast mac
1709      * entry to include this dpn tep ip and schedule the job to put ucast macs
1710      * once the physical locator is programmed in device
1711      *
1712      * @param elanName
1713      *            the elan name
1714      * @param lstElanInterfaceNames
1715      *            the lst Elan interface names
1716      * @param dpnId
1717      *            the dpn id
1718      * @param externalNodeId
1719      *            the external node id
1720      */
1721     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, BigInteger dpnId,
1722             NodeId externalNodeId) {
1723         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1724                 externalNodeId.getValue());
1725         if (elanL2GwDevice == null) {
1726             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1727             return;
1728         }
1729         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1730         if (dpnTepIp == null) {
1731             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1732                     externalNodeId);
1733             return;
1734         }
1735
1736         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1737         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1738                 LogicalDatastoreType.OPERATIONAL);
1739         boolean phyLocAlreadyExists =
1740                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1741                 dpnTepIp);
1742         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1743                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1744         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1745
1746         if (phyLocAlreadyExists) {
1747             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1748             return;
1749         }
1750         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1751         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1752     }
1753
1754     /**
1755      * Validate external tunnel state event.
1756      *
1757      * @param externalTunnel
1758      *            the external tunnel
1759      * @param intrf
1760      *            the intrf
1761      * @return true, if successful
1762      */
1763     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1764         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1765             String srcDevice = externalTunnel.getDestinationDevice();
1766             String destDevice = externalTunnel.getSourceDevice();
1767             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1768                     LogicalDatastoreType.CONFIGURATION);
1769             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1770                     otherEndPointExtTunnel);
1771             if (otherEndPointExtTunnel != null) {
1772                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1773                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1774                 if (otherEndPointInterfaceOperational) {
1775                     return true;
1776                 } else {
1777                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1778                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1779                 }
1780             }
1781         }
1782         return false;
1783     }
1784
1785     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1786         List<MatchInfo> mkMatches = new ArrayList<>();
1787         // Matching metadata
1788         mkMatches.add(
1789                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1790         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(lportTag)));
1791         return mkMatches;
1792     }
1793
1794     @Override
1795     protected ElanInterfaceManager getDataTreeChangeListener() {
1796         return this;
1797     }
1798
1799     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1800                                              BigInteger dpId) {
1801         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1802         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1803             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1804                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1805         try {
1806             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1807         } catch (InterruptedException e) {
1808             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1809         }
1810     }
1811 }