elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 142 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.opendaylight.yangtools.yang.common.Uint32;
 145 import org.opendaylight.yangtools.yang.common.Uint64;
 146 import org.slf4j.Logger;
 147 import org.slf4j.LoggerFactory;
 148
 149 /**
 150  * Class in charge of handling creations, modifications and removals of
 151  * ElanInterfaces.
 152  *
 153  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 154  */
 155 @Singleton
 156 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 157         implements RecoverableListener {
 158     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 159     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 160     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 161     private static final boolean SH_FLAG_SET = true;
 162     private static final boolean SH_FLAG_UNSET = false;
 163
 164     private final DataBroker broker;
 165     private final ManagedNewTransactionRunner txRunner;
 166     private final IMdsalApiManager mdsalManager;
 167     private final IInterfaceManager interfaceManager;
 168     private final IdManagerService idManager;
 169     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 170     private final INeutronVpnManager neutronVpnManager;
 171     private final ElanItmUtils elanItmUtils;
 172     private final ElanEtreeUtils elanEtreeUtils;
 173     private final ElanL2GatewayUtils elanL2GatewayUtils;
 174     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 175     private final ElanUtils elanUtils;
 176     private final JobCoordinator jobCoordinator;
 177     private final ElanInstanceCache elanInstanceCache;
 178     private final ElanInterfaceCache elanInterfaceCache;
 179     private final ElanGroupCache elanGroupCache;
 180
 181     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 182         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 183
 184     @Inject
 185     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 186                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 187                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 188                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 189                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 190                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 191                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 192                                 final ElanInstanceCache elanInstanceCache,
 193                                 final ElanInterfaceCache elanInterfaceCache,
 194                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 195                                 ElanGroupCache elanGroupCache,
 196                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 197         super(ElanInterface.class, ElanInterfaceManager.class);
 198         this.broker = dataBroker;
 199         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 200         this.idManager = managerService;
 201         this.mdsalManager = mdsalApiManager;
 202         this.interfaceManager = interfaceManager;
 203         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 204         this.neutronVpnManager = neutronVpnManager;
 205         this.elanItmUtils = elanItmUtils;
 206         this.elanEtreeUtils = elanEtreeUtils;
 207         this.elanL2GatewayUtils = elanL2GatewayUtils;
 208         this.elanUtils = elanUtils;
 209         this.jobCoordinator = jobCoordinator;
 210         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 211         this.elanInstanceCache = elanInstanceCache;
 212         this.elanInterfaceCache = elanInterfaceCache;
 213         this.elanGroupCache = elanGroupCache;
 214         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 215     }
 216
 217     @Override
 218     @PostConstruct
 219     public void init() {
 220         registerListener();
 221     }
 222
 223     @Override
 224     public void registerListener() {
 225         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 226     }
 227
 228     @Override
 229     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 230         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 231     }
 232
 233     @Override
 234     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 235         String interfaceName = del.getName();
 236         String elanInstanceName = del.getElanInstanceName();
 237         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 238         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 239         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 240             elanInterfaces.remove(del);
 241             if (elanInterfaces.isEmpty()) {
 242                 unProcessedElanInterfaces.remove(elanInstanceName);
 243             }
 244         }
 245         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 246         /*
 247          * Handling in case the elan instance is deleted.If the Elan instance is
 248          * deleted, there is no need to explicitly delete the elan interfaces
 249          */
 250         if (elanInfo == null) {
 251             return;
 252         }
 253         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 254         if (interfaceInfo == null && elanInfo.isExternal()) {
 255             // In deleting external network, the underlying ietf Inteface might have been removed
 256             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 257             // from Operational DS instead
 258             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 259         }
 260         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 261                 interfaceName, interfaceInfo, this);
 262         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 263     }
 264
 265     private static class RemoveElanInterfaceHolder {
 266         boolean isLastElanInterface = false;
 267         Uint64 dpId = null;
 268     }
 269
 270     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 271     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 272             InterfaceInfo interfaceInfo) {
 273         String elanName = elanInfo.getElanInstanceName();
 274         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 275         Uint32 elanTag = elanInfo.getElanTag();
 276         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 277         List<ListenableFuture<Void>> futures = new ArrayList<>();
 278         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 279         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 280             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 281             if (elanState == null) {
 282                 return;
 283             }
 284             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 285                 List<String> elanInterfaces = elanState.getElanInterfaces();
 286                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 287                     holder.isLastElanInterface = true;
 288                 }
 289                 if (interfaceInfo != null) {
 290                     holder.dpId = interfaceInfo.getDpId();
 291                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 292                         interfaceName, elanTag, interfaceTx);
 293                     /*
 294                      * If there are not elan ports, remove the unknown dmac, terminating
 295                      * service table flows, remote/local bc group
 296                      */
 297                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 298                         || dpnInterfaces.getInterfaces().isEmpty()) {
 299                         // No more Elan Interfaces in this DPN
 300                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 301                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 302                             holder.dpId);
 303                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 304                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 305                         }
 306                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 307                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 308                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 309                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 310                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 311                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 312                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 313                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 314                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 315                             }
 316                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 317                         }
 318                     } else {
 319                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 320                     }
 321                 }
 322             }));
 323         }));
 324         futures.forEach(ElanUtils::waitForTransactionToComplete);
 325
 326         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 327                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 328         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 329                 ElanConstants.JOB_MAX_RETRIES);
 330
 331         return futures;
 332     }
 333
 334     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 335             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 336             throws ExecutionException, InterruptedException {
 337         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 338         if (etreeLeafTag != null) {
 339             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 340             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 341         }
 342     }
 343
 344     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 345             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 346             throws ExecutionException, InterruptedException {
 347         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 348         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 349     }
 350
 351     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 352             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 353             throws ExecutionException, InterruptedException {
 354         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 355         if (etreeInstance != null) {
 356             Uint64 dpnId = interfaceInfo.getDpId();
 357             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 358             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 359             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 360         }
 361     }
 362
 363     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 364             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 365             throws ExecutionException, InterruptedException {
 366         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 367         if (etreeInstance != null) {
 368             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 369             Uint64 dpnId = interfaceInfo.getDpId();
 370             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 371             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 372             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 373         }
 374     }
 375
 376     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 377             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 378         String elanName = elanInfo.getElanInstanceName();
 379         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 380         if (elanState == null) {
 381             return elanState;
 382         }
 383         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 384         List<String> elanInterfaces = new ArrayList<>();
 385         if (existingElanInterfaces != null) {
 386             elanInterfaces.addAll(existingElanInterfaces);
 387         }
 388         boolean isRemoved = elanInterfaces.remove(interfaceName);
 389         if (!isRemoved) {
 390             return elanState;
 391         }
 392
 393         if (elanInterfaces.isEmpty()) {
 394             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 395             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 396             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 397             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 398         } else {
 399             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 400                     .withKey(new ElanKey(elanName)).build();
 401             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 402         }
 403         return elanState;
 404     }
 405
 406     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 407             throws ReadFailedException {
 408         // removing the ElanInterface from the config data_store if interface is
 409         // not present in Interface config DS
 410         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 411             interfaceName) == null
 412                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 413             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 414         }
 415     }
 416
 417     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 418             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 419         String elanName = elanInfo.getElanInstanceName();
 420         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 421         List<ListenableFuture<Void>> futures = new ArrayList<>();
 422         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 423             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 424                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 425                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 426                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 427                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 428                 if (interfaceInfo != null) {
 429                     if (existingElanInterfaceMac.isPresent()) {
 430                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 431                         if (existingMacEntries != null) {
 432                             List<PhysAddress> macAddresses = new ArrayList<>();
 433                             for (MacEntry macEntry : existingMacEntries) {
 434                                 PhysAddress macAddress = macEntry.getMacAddress();
 435                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 436                                         macAddress.getValue(), interfaceName);
 437                                 Optional<MacEntry> macEntryOptional =
 438                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 439                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 440                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 441                                 }
 442                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 443                                         interfaceName, elanName, macAddress);
 444                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 445                                 macAddresses.add(macAddress);
 446                             }
 447
 448                             // Removing all those MACs from External Devices belonging
 449                             // to this ELAN
 450                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 451                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 452                             }
 453                         }
 454                     }
 455                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 456                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 457                 } else if (existingElanInterfaceMac.isPresent()) {
 458                     // Interface does not exist in ConfigDS, so lets remove everything
 459                     // about that interface related to Elan
 460                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 461                     if (macEntries != null) {
 462                         for (MacEntry macEntry : macEntries) {
 463                             PhysAddress macAddress = macEntry.getMacAddress();
 464                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 465                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 466                             }
 467                         }
 468                     }
 469                 }
 470                 if (existingElanInterfaceMac.isPresent()) {
 471                     interfaceTx.delete(elanInterfaceId);
 472                 }
 473                 unbindService(interfaceName, flowTx);
 474                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 475             }));
 476         }));
 477         return futures;
 478     }
 479
 480     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 481                                                                          String interfaceName, Uint32 elanTag,
 482                                                                          TypedReadWriteTransaction<Operational> tx)
 483             throws ExecutionException, InterruptedException {
 484         // FIXME: pass in and use ElanInstanceKey instead?
 485         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 486         lock.lock();
 487         try {
 488             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 489             if (dpnInterfaces != null) {
 490                 List<String> interfaceLists = null;
 491                 if (dpnInterfaces.getInterfaces() != null) {
 492                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 493                 }
 494                 if (interfaceLists != null) {
 495                     interfaceLists.remove(interfaceName);
 496                 }
 497
 498                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 499                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 500                     deleteElanDpnInterface(elanName, dpId, tx);
 501                 } else {
 502                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 503                 }
 504             }
 505             return dpnInterfaces;
 506         } finally {
 507             lock.unlock();
 508         }
 509     }
 510
 511     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 512         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 513         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 514             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 515                 Uint64 currentDpId = dpnInterface.getDpId();
 516                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 517                     for (String elanInterface : dpnInterface.getInterfaces()) {
 518                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 519                         if (macs == null || macs.getMacEntry() == null) {
 520                             continue;
 521                         }
 522                         for (MacEntry mac : macs.getMacEntry()) {
 523                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 524                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 525                         }
 526                     }
 527                 }
 528             }
 529         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 530     }
 531
 532     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 533             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 534         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 535         if (etreeLeafTag != null) {
 536             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(),
 537                 currentDpId, mac, confTx);
 538         }
 539     }
 540
 541     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 542             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 543         mdsalManager
 544                 .removeFlow(confTx, dpId,
 545                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 546                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 547                                         mac.getMacAddress().getValue(), elanTag)));
 548     }
 549
 550     /*
 551     * Possible Scenarios for update
 552     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 553         then all {1,2,3,4} should be removed
 554
 555         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 556         then all {1,2,3,4} should be added
 557
 558         c. if orig = {1,2,3,4} updated={2,3,4}
 559         then 1 should be removed
 560
 561         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 562         then we should just add 5
 563
 564         e. if orig = {1,2,3,4} updated={2,3,4,5}
 565         then 1 should be removed , 5 should be added
 566     * */
 567     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 568     @Override
 569     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 570         // updating the static-Mac Entries for the existing elanInterface
 571         String elanName = update.getElanInstanceName();
 572         String interfaceName = update.getName();
 573         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 574         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 575
 576         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 577         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 578         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 579         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 580
 581         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 582                 deletedEntry.getMacAddress()));
 583
 584         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 585         * Hence add the macentries for the same.*/
 586         for (StaticMacEntries staticMacEntry : updatedEntries) {
 587             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 588                     staticMacEntry.getMacAddress());
 589             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 590                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 591                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 592                     if (existingMacEntry.isPresent()) {
 593                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 594                                 existingMacEntry.get(), elanName);
 595                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 596                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 597                             tx);
 598                     } else {
 599                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 600                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 601                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 602                             elanName, interfaceName, staticMacEntry, tx);
 603                     }
 604                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 605         }
 606     }
 607
 608     @Override
 609     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 610         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 611         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 612             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 613             String interfaceName = elanInterfaceAdded.getName();
 614             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 615             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 616             if (interfaceInfo == null) {
 617                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 618                 return;
 619             }
 620             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 621
 622             if (elanInstance == null) {
 623                 // Add the ElanInstance in the Configuration data-store
 624                 List<String> elanInterfaces = new ArrayList<>();
 625                 elanInterfaces.add(interfaceName);
 626                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 627                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 628                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 629                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 630             }
 631
 632             Uint32 elanTag = elanInstance.getElanTag();
 633             // If elan tag is not updated, then put the elan interface into
 634             // unprocessed entry map and entry. Let entries
 635             // in this map get processed during ELAN update DCN.
 636             if (elanTag == null || elanTag.longValue() == 0L) {
 637                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 638                 if (elanInterfaces == null) {
 639                     elanInterfaces = new ConcurrentLinkedQueue<>();
 640                 }
 641                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 642                     elanInterfaces.add(elanInterfaceAdded);
 643                 }
 644                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 645                 return;
 646             }
 647             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 648                     interfaceInfo, elanInstance, this);
 649             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 650         }), LOG, "Error processing added ELAN interface");
 651     }
 652
 653     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 654         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 655         List<ListenableFuture<Void>> futures = new ArrayList<>();
 656         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 657         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 658             return futures;
 659         }
 660         for (ElanInterface elanInterface : elanInterfaces) {
 661             String interfaceName = elanInterface.getName();
 662             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 663             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 664         }
 665         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 666         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 667                 elanInstance.getElanInstanceName());
 668         return futures;
 669     }
 670
 671     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 672             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 673         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 674                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 675         List<DpnInterfaces> dpnInterfaceLists = null;
 676         if (elanDpnInterfacesList != null) {
 677             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 678         }
 679         if (dpnInterfaceLists == null) {
 680             dpnInterfaceLists = new ArrayList<>();
 681         }
 682         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 683             Uint64 dstDpId = interfaceInfo.getDpId();
 684             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 685                 continue;
 686             }
 687             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 688             for (String remoteIf : remoteElanInterfaces) {
 689                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 690                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 691                 if (elanIfMac == null || remoteInterface == null) {
 692                     continue;
 693                 }
 694                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 695                 if (remoteMacEntries != null) {
 696                     for (MacEntry macEntry : remoteMacEntries) {
 697                         String macAddress = macEntry.getMacAddress().getValue();
 698                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 699                                 dstDpId, elanInstance.getElanInstanceName());
 700                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 701                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 702                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 703                     }
 704                 }
 705             }
 706         }
 707     }
 708
 709     private static class AddElanInterfaceHolder {
 710         private DpnInterfaces dpnInterfaces = null;
 711         private boolean isFirstInterfaceInDpn = false;
 712         private Uint64 dpId;
 713     }
 714
 715     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 716     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 717             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 718         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 719         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 720         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 721
 722         String interfaceName = elanInterface.getName();
 723         String elanInstanceName = elanInterface.getElanInstanceName();
 724         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 725         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 726
 727         List<ListenableFuture<Void>> futures = new ArrayList<>();
 728         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 729         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 730             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 731             if (elanInfo == null) {
 732                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 733                 List<String> elanInterfaces = new ArrayList<>();
 734                 elanInterfaces.add(interfaceName);
 735                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 736                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 737                         operTx)));
 738             } else {
 739                 createElanStateList(elanInstanceName, interfaceName, operTx);
 740             }
 741             // Specific actions to the DPN where the ElanInterface has been added,
 742             // for example, programming the
 743             // External tunnel table if needed or adding the ElanInterface to the
 744             // DpnInterfaces in the operational DS.
 745             holder.dpId = interfaceInfo.getDpId();
 746             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 747                 // FIXME: use elanInstaince.key() instead?
 748                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 749                 lock.lock();
 750                 try {
 751                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 752                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 753                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 754                     if (ElanUtils.isVlan(elanInstance)) {
 755                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 756                             elanInstanceName, existingElanDpnInterfaces);
 757                     } else {
 758                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 759                     }
 760                     if (holder.isFirstInterfaceInDpn) {
 761                         // ELAN's 1st ElanInterface added to this DPN
 762                         if (!existingElanDpnInterfaces.isPresent()) {
 763                             holder.dpnInterfaces =
 764                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 765                         } else {
 766                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 767                             List<String> elanInterfaces =
 768                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 769                             elanInterfaces.add(interfaceName);
 770                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 771                                 elanInterfaces, operTx);
 772                         }
 773                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 774                                 interfaceName, elanInstanceName, holder.dpId);
 775                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 776                         // table, but only if Elan has VNI
 777                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 778                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 779                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 780                         }
 781                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 782                             interfaceName);
 783                     } else {
 784                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 785                         List<String> elanInterfaces =
 786                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 787                         elanInterfaces.add(interfaceName);
 788                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 789                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 790                                 interfaceName);
 791                         }
 792                         holder.dpnInterfaces =
 793                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 794                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 795                                 interfaceName, elanInstanceName, holder.dpId);
 796                     }
 797                 } finally {
 798                     lock.unlock();
 799                 }
 800             }
 801
 802             // add code to install Local/Remote BC group, unknow DMAC entry,
 803             // terminating service table flow entry
 804             // call bindservice of interfacemanager to create ingress table flow
 805             // enty.
 806             // Add interface to the ElanInterfaceForwardingEntires Container
 807             createElanInterfaceTablesList(interfaceName, operTx);
 808         }));
 809         futures.forEach(ElanUtils::waitForTransactionToComplete);
 810         futures.add(
 811             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 812                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 813                     holder.isFirstInterfaceInDpn, confTx))));
 814
 815         // add the vlan provider interface to remote BC group for the elan
 816         // for internal vlan networks
 817         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 818             if (interfaceManager.isExternalInterface(interfaceName)) {
 819                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 820                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 821             }
 822         }
 823         if (holder.isFirstInterfaceInDpn) {
 824             // ELAN's 1st ElanInterface added to this DPN
 825             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 826             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 827                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 828                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 829             }));
 830         } else {
 831             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 832         }
 833
 834         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 835                 holder.isFirstInterfaceInDpn, elanInterface);
 836         return futures;
 837     }
 838
 839     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 840     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 841             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 842         String elanInstanceName = elanInstance.getElanInstanceName();
 843         String interfaceName = elanInterface.getName();
 844         List<ListenableFuture<Void>> futures = new ArrayList<>();
 845         Uint64 dpId = interfaceInfo.getDpId();
 846         boolean isInterfaceOperational = isOperational(interfaceInfo);
 847         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 848             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 849                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 850                     isFirstInterfaceInDpn, confTx);
 851
 852                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 853                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 854
 855                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 856                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 857                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 858                             staticMacEntry.getMacAddress());
 859                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 860                             LogicalDatastoreType.OPERATIONAL, macId);
 861                         if (existingMacEntry.isPresent()) {
 862                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 863                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 864                                 existingMacEntry.get(), operTx);
 865                         } else {
 866                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 867                                 interfaceName, staticMacEntry, operTx);
 868                         }
 869
 870                         if (isInterfaceOperational) {
 871                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 872                             // DPNs
 873                             String macAddress = staticMacEntry.getMacAddress().getValue();
 874                             LOG.info(
 875                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 876                                     + " {}",
 877                                 macAddress, elanInstanceName, interfaceName);
 878                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 879                                     interfaceName, elanInstanceName, macAddress);
 880                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 881                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 882                         }
 883                     }
 884
 885                     if (isInterfaceOperational) {
 886                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 887                         // on purpose.
 888                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 889                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 890                         }
 891                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 892                             staticMacAddresses);
 893                     }
 894                 }
 895             }));
 896         }));
 897         futures.forEach(ElanUtils::waitForTransactionToComplete);
 898         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 899             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 900             try {
 901                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 902                 if (neutronPort != null) {
 903                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 904                 }
 905             } catch (IllegalArgumentException ex) {
 906                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 907             }
 908         }
 909         return futures;
 910     }
 911
 912     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 913             PhysAddress physAddress) {
 914         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 915         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 916         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 917                 LogicalDatastoreType.OPERATIONAL, macId);
 918
 919         if (!existingMacEntry.isPresent()) {
 920             return;
 921         }
 922
 923         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 924                 .withKey(new MacEntryKey(physAddress)).build();
 925         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 926                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 927     }
 928
 929     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 930             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 931         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 932         if (!existingElanDpnInterfaces.isPresent()) {
 933             return true;
 934         }
 935         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 936             return false;
 937         }
 938         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 939         int dummyInterfaceCount =  0;
 940         List<String> interfaces = dpnInterfaces.getInterfaces();
 941         if (interfaces == null) {
 942             return true;
 943         }
 944         if (interfaces.contains(routerPortUuid)) {
 945             dummyInterfaceCount++;
 946         }
 947         if (interfaces.contains(elanInstanceName)) {
 948             dummyInterfaceCount++;
 949         }
 950         return interfaces.size() == dummyInterfaceCount;
 951     }
 952
 953     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 954             PhysAddress physAddress) {
 955         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 956                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 957     }
 958
 959     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 960             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 961         if (!isOperational(interfaceInfo)) {
 962             LOG.warn("Interface {} is not operational", elanInterface.getName());
 963             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 964                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 965             return;
 966         }
 967         Uint64 dpId = interfaceInfo.getDpId();
 968         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 969             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 970         }
 971         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 972         if (isFirstInterfaceInDpn) {
 973             // Terminating Service , UnknownDMAC Table.
 974             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 975             // for ELAN Instance is VxLAN
 976             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 977                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 978             }
 979             setupUnknownDMacTable(elanInstance, dpId, confTx);
 980             /*
 981              * Install remote DMAC flow. This is required since this DPN is
 982              * added later to the elan instance and remote DMACs of other
 983              * interfaces in this elan instance are not present in the current
 984              * dpn.
 985              */
 986             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 987                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 988                         elanInstance.getElanInstanceName());
 989                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 990             }
 991         }
 992         // bind the Elan service to the Interface
 993         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 994     }
 995
 996     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 997             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 998         if (!isOperational(interfaceInfo)) {
 999             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
1000                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1001             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1002                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1003             return;
1004         }
1005         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1006                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1007         // LocalBroadcast Group creation with elan-Interfaces
1008         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1009                 dpnInterfaces.getDpId());
1010         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1011         if (isFirstInterfaceInDpn) {
1012             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1013             try {
1014                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1015             } catch (InterruptedException e1) {
1016                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1017             }
1018         }
1019     }
1020
1021     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1022         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1023         return InstanceIdentifier.builder(Nodes.class)
1024                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1025                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1026                 .augmentation(FlowCapableNode.class)
1027                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1028     }
1029
1030     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1031                                                               InterfaceInfo interfaceInfo,
1032                                                               DpnInterfaces dpnInterfaces,
1033                                                               boolean isFirstInterfaceInDpn,
1034                                                               ElanInterface elanInterface) {
1035         if (!isOperational(interfaceInfo)) {
1036             LOG.debug("Interface {} is not operational", elanInterface.getName());
1037             return;
1038         }
1039         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1040         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1041                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1042         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1043         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1044             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1045         });
1046     }
1047
1048     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1049             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1050         int ifTag = interfaceInfo.getInterfaceTag();
1051         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1052                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 9, elanInfo.getElanInstanceName(), 0,
1053                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1054                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1055                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1056
1057         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1058         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1059                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1060
1061         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1062                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1063                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1064                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1065
1066         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1067         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1068                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1069     }
1070
1071     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1072             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1073         int ifTag = interfaceInfo.getInterfaceTag();
1074         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1075             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1076
1077         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1078
1079         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1080             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 10, elanInfo.getElanInstanceName(), 0,
1081             0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1082             getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1083
1084         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1085     }
1086
1087     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1088         List<MatchInfo> mkMatches = new ArrayList<>();
1089         MatchInfo match = new MatchTunnelId(vni);
1090         mkMatches.add(match);
1091         return mkMatches;
1092     }
1093
1094     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1095         List<InstructionInfo> mkInstructions = new ArrayList<>();
1096         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1097         return mkInstructions;
1098     }
1099
1100     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1101         List<MatchInfo> mkMatches = new ArrayList<>();
1102         // Matching metadata
1103         mkMatches.add(new MatchMetadata(
1104                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1105         return mkMatches;
1106     }
1107
1108     /**
1109      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1110      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1111      *
1112      * @param elanTag
1113      *            elanTag to be written in metadata when flow is selected
1114      * @return the instructions ready to be installed in a flow
1115      */
1116     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1117         List<InstructionInfo> mkInstructions = new ArrayList<>();
1118         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1119                 ElanHelper.getElanMetadataMask()));
1120         /* applicable for EXTERNAL_TUNNEL_TABLE only
1121         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1122         */
1123         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1124         return mkInstructions;
1125     }
1126
1127     // Install DMAC entry on dst DPN
1128     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1129             Uint64 dstDpId) {
1130         String interfaceName = interfaceInfo.getInterfaceName();
1131         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1132         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1133             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1134             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1135                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1136                     for (MacEntry macEntry : macEntries) {
1137                         String macAddress = macEntry.getMacAddress().getValue();
1138                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1139                             interfaceName);
1140                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1141                             interfaceInfo.getDpId())) {
1142                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1143                                 macAddress);
1144                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1145                         }
1146                     }
1147                 })));
1148         }
1149         return emptyList();
1150     }
1151
1152     private static void createDropBucket(List<Bucket> listBucket) {
1153         List<Action> actionsInfos = new ArrayList<>();
1154         actionsInfos.add(new ActionDrop().buildAction());
1155         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1156                 MDSALUtil.WATCH_GROUP);
1157         listBucket.add(dropBucket);
1158     }
1159
1160     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1161             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1162         if (!isOperational(interfaceInfo)) {
1163             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1164                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1165             return;
1166         }
1167         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1168         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1169     }
1170
1171     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1172             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1173         List<Bucket> listBucket = new ArrayList<>();
1174         int bucketId = 0;
1175         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1176
1177         List<String> interfaces = new ArrayList<>();
1178         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1179             interfaces = newDpnInterface.getInterfaces();
1180         }
1181         for (String ifName : interfaces) {
1182             // In case if there is a InterfacePort in the cache which is not in
1183             // operational state, skip processing it
1184             InterfaceInfo ifInfo = interfaceManager
1185                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1186             if (!isOperational(ifInfo)) {
1187                 continue;
1188             }
1189
1190             if (!interfaceManager.isExternalInterface(ifName)) {
1191                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1192                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1193                 bucketId++;
1194             }
1195         }
1196
1197         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1198                 MDSALUtil.buildBucketLists(listBucket));
1199         LOG.trace("installing the localBroadCast Group:{}", group);
1200         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1201     }
1202
1203     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1204             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1205         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1206         if (etreeInstance != null) {
1207             List<Bucket> listBucket = new ArrayList<>();
1208             int bucketId = 0;
1209
1210             List<String> interfaces = new ArrayList<>();
1211             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1212                 interfaces = newDpnInterface.getInterfaces();
1213             }
1214             for (String ifName : interfaces) {
1215                 // In case if there is a InterfacePort in the cache which is not
1216                 // in
1217                 // operational state, skip processing it
1218                 InterfaceInfo ifInfo = interfaceManager
1219                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1220                 if (!isOperational(ifInfo)) {
1221                     continue;
1222                 }
1223
1224                 if (!interfaceManager.isExternalInterface(ifName)) {
1225                     // only add root interfaces
1226                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1227                 }
1228             }
1229
1230             if (listBucket.isEmpty()) { // No Buckets
1231                 createDropBucket(listBucket);
1232             }
1233
1234             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1235             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1236             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1237                     MDSALUtil.buildBucketLists(listBucket));
1238             LOG.trace("installing the localBroadCast Group:{}", group);
1239             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1240         }
1241     }
1242
1243     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1244             InterfaceInfo ifInfo) {
1245         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1246         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1247             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1248                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1249             bucketId++;
1250         }
1251         return bucketId;
1252     }
1253
1254     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1255             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1256             throws ExecutionException, InterruptedException {
1257         Uint64 dpnId = interfaceInfo.getDpId();
1258         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1259         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1260         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1261     }
1262
1263     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1264             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1265             throws ExecutionException, InterruptedException {
1266         Uint64 dpnId = interfaceInfo.getDpId();
1267         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1268         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1269         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1270     }
1271
1272     /**
1273      * Installs a flow in the External Tunnel table consisting in translating
1274      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1275      * elanTag that will be used later in the ELANs pipeline.
1276      * @param dpnId the dpn id
1277      */
1278     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1279             TypedWriteTransaction<Configuration> confTx) {
1280         Uint32 elanTag = elanInfo.getElanTag();
1281         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1282                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1283                 elanInfo.getElanInstanceName(), // flowName
1284                 0, // idleTimeout
1285                 0, // hardTimeout
1286                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1287                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1288                 getInstructionsIntOrExtTunnelTable(elanTag));
1289
1290         mdsalManager.addFlow(confTx, flowEntity);
1291     }
1292
1293     /**
1294      * Removes, from External Tunnel table, the flow that translates from VNI to
1295      * elanTag. Important: ensure this method is only called whenever there is
1296      * no other ElanInterface in the specified DPN
1297      *
1298      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1299      */
1300     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1301             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1302         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1303         // last ElanInstance plus
1304         // adding a new one does (almost at the same time) are executed in that
1305         // exact order
1306
1307         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1308         FlowEntity flowEntity = new FlowEntityBuilder()
1309             .setDpnId(dpnId)
1310             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1311             .setFlowId(flowId)
1312             .build();
1313         mdsalManager.removeFlow(confTx, flowEntity);
1314     }
1315
1316     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1317             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1318         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1319         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1320     }
1321
1322     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1323             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1324         List<? extends MatchInfoBase> listMatchInfoBase;
1325         List<InstructionInfo> instructionInfos;
1326         Uint32 serviceId;
1327         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1328             serviceId = elanTag;
1329             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1330             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1331         } else {
1332             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1333             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1334             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1335         }
1336         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1337                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1338                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1339                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1340                 listMatchInfoBase, instructionInfos);
1341         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1342         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1343                 elanInfo.getElanInstanceName(), elanTag);
1344     }
1345
1346     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1347             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1348         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1349         if (etreeInstance != null) {
1350             setupTerminateServiceTable(elanInfo, dpId,
1351                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1352         }
1353     }
1354
1355     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1356             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1357         long elanTag = elanInfo.getElanTag().toJava();
1358         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1359         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1360         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1361     }
1362
1363     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1364             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1365         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1366         if (etreeLeafTag != null) {
1367             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1368             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1369             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1370         }
1371     }
1372
1373     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1374             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1375         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1376                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1377                 5, elanInfo.getElanInstanceName(), 0, 0,
1378                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1379                 getMatchesForElanTag(elanTag, /* SH flag */false),
1380                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1381
1382         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1383         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1384                 dpId, elanInfo.getElanInstanceName(), elanTag);
1385     }
1386
1387     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1388             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1389         // only if ELAN can connect to external network, perform the following
1390
1391         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1392             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1393                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1394                     5, elanInfo.getElanInstanceName(), 0, 0,
1395                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1396                     getMatchesForElanTag(elanTag, /* SH flag */true),
1397                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1398             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1399             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1400                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1401         }
1402     }
1403
1404
1405     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1406             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1407             throws ExecutionException, InterruptedException {
1408         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1409                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1410         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1411
1412         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1413             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1414                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1415                     .build();
1416             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1417         }
1418     }
1419
1420     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1421         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1422     }
1423
1424     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1425             TypedWriteTransaction<Configuration> tx) {
1426         if (isStandardElanService(elanInterface)) {
1427             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1428                     elanInterface.getName(), lportTag, tx);
1429         } else { // Etree service
1430             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1431         }
1432     }
1433
1434     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1435             TypedWriteTransaction<Configuration> tx) {
1436         int instructionKey = 0;
1437         List<Instruction> instructions = new ArrayList<>();
1438         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1439                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1440
1441         List<Action> actions = new ArrayList<>();
1442         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1443                 lportTag).buildAction());
1444         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1445                 elanTag).buildAction());
1446         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1447
1448         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1449                 ++instructionKey));
1450
1451         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1452         BoundServices serviceInfo = ElanUtils.getBoundServices(
1453                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1454                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1455         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1456         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1457                 bindServiceId);
1458         if (!existingElanService.isPresent()) {
1459             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1460             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1461         }
1462     }
1463
1464     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1465             TypedWriteTransaction<Configuration> tx) {
1466         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1467             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1468                     lportTag, tx);
1469         } else {
1470             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1471             if (etreeInstance == null) {
1472                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1473                         elanInterface.getName(), elanInfo.getElanInstanceName());
1474             } else {
1475                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1476                         elanInterface.getName(), lportTag, tx);
1477             }
1478         }
1479     }
1480
1481     private static boolean isStandardElanService(ElanInterface elanInterface) {
1482         return elanInterface.augmentation(EtreeInterface.class) == null;
1483     }
1484
1485     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1486             throws ExecutionException, InterruptedException {
1487         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1488         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1489         if (tx.read(bindServiceId).get().isPresent()) {
1490             tx.delete(bindServiceId);
1491         }
1492     }
1493
1494     private static String getFlowRef(long tableId, long elanTag) {
1495         return String.valueOf(tableId) + elanTag;
1496     }
1497
1498     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1499         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1500                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1501     }
1502
1503     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1504         return String.valueOf(tableId) + elanTag + shFlag;
1505     }
1506
1507     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1508         List<Action> listAction = new ArrayList<>();
1509         int actionKey = 0;
1510         listAction.add(
1511             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1512         actionKey++;
1513         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1514         return listAction;
1515     }
1516
1517     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1518             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1519         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1520                 .withKey(new DpnInterfacesKey(dpId)).build();
1521         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1522                 CREATE_MISSING_PARENTS);
1523         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1524                 interfaceNames);
1525         return dpnInterface;
1526     }
1527
1528     /**
1529      * Delete elan dpn interface from operational DS.
1530      *
1531      * @param elanInstanceName
1532      *            the elan instance name
1533      * @param dpId
1534      *            the dp id
1535      */
1536     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1537             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1538         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1539                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1540         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1541         if (dpnInterfaces.isPresent()) {
1542             tx.delete(dpnInterfacesId);
1543         }
1544     }
1545
1546     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1547             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1548         List<String> interfaceNames = new ArrayList<>();
1549         interfaceNames.add(interfaceName);
1550         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1551                 .withKey(new DpnInterfacesKey(dpId)).build();
1552         return dpnInterface;
1553     }
1554
1555     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1556             throws ExecutionException, InterruptedException {
1557         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1558                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1559         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1560         // Adding new Elan Interface Port to the operational DataStore without
1561         // Static-Mac Entries..
1562         if (!interfaceMacTables.isPresent()) {
1563             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1564                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1565             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1566                     CREATE_MISSING_PARENTS);
1567             LOG.trace("Created interface MAC table for interface: {}", interfaceName);
1568         }
1569     }
1570
1571     private static void createElanStateList(String elanInstanceName, String interfaceName,
1572             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1573         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1574         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1575         // Adding new Elan Interface Port to the operational DataStore without
1576         // Static-Mac Entries..
1577         if (elanInterfaceLists.isPresent()) {
1578             List<String> interfaceLists = new ArrayList<>();
1579             interfaceLists.add(interfaceName);
1580             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1581             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1582                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1583             }
1584
1585             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1586                     .withKey(new ElanKey(elanInstanceName)).build();
1587             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1588             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1589                     interfaceLists);
1590         }
1591     }
1592
1593     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1594         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1595     }
1596
1597     @SuppressWarnings("checkstyle:IllegalCatch")
1598     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1599         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1600         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1601                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1602         if (dpnInterfaceLists == null) {
1603             return;
1604         }
1605         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1606         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1607             int cnt = 0;
1608             String elanName = elanDpns.getElanInstanceName();
1609             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1610             if (elanInfo == null) {
1611                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1612                         + "skipping this ELAN for tunnel handling", elanName);
1613                 continue;
1614             }
1615             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1616                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1617                 continue;
1618             }
1619             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1620             if (dpnInterfaces == null) {
1621                 continue;
1622             }
1623             DpnInterfaces dstDpnIf = null;
1624             for (DpnInterfaces dpnIf : dpnInterfaces) {
1625                 Uint64 dpnIfDpId = dpnIf.getDpId();
1626                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1627                     cnt++;
1628                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1629                     cnt++;
1630                     dstDpnIf = dpnIf;
1631                 }
1632             }
1633             if (cnt == 2) {
1634                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1635                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1636                 jobCoordinator.enqueueJob(elanName, () -> {
1637                     // update Remote BC Group
1638                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1639                     try {
1640                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1641                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1642                                 confTx)).get();
1643                     } catch (RuntimeException e) {
1644                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1645                     }
1646                     Set<String> interfaceLists = new HashSet<>();
1647                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1648                     for (String ifName : interfaceLists) {
1649                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1650                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1651                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1652                             if (isOperational(interfaceInfo)) {
1653                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1654                             }
1655                             return emptyList();
1656                         }, ElanConstants.JOB_MAX_RETRIES);
1657                     }
1658                     return emptyList();
1659                 }, ElanConstants.JOB_MAX_RETRIES);
1660             }
1661
1662         }
1663     }
1664
1665     /**
1666      * Handle external tunnel state event.
1667      *
1668      * @param externalTunnel
1669      *            the external tunnel
1670      * @param intrf
1671      *            the interface
1672      */
1673     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1674         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1675             return;
1676         }
1677         // dpId/externalNodeId will be available either in source or destination
1678         // based on the tunnel end point
1679         Uint64 dpId = null;
1680         NodeId externalNodeId = null;
1681         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1682             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1683             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1684         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1685             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1686             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1687         }
1688         if (dpId == null || externalNodeId == null) {
1689             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1690             return;
1691         }
1692
1693         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1694         if (dpnInterfaceLists == null) {
1695             return;
1696         }
1697         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1698         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1699             String elanName = elanDpns.getElanInstanceName();
1700             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1701
1702             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1703             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1704                     || dpnInterfaces.getInterfaces().isEmpty()) {
1705                 continue;
1706             }
1707             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1708
1709             final Uint64 finalDpId = dpId;
1710             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1711                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1712                 "Error setting up ELAN BGs");
1713             // install L2gwDevices local macs in dpn.
1714             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1715             // Install dpn macs on external device
1716             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1717                     externalNodeId);
1718         }
1719         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1720     }
1721
1722     /**
1723      * Installs dpn macs in external device. first it checks if the physical
1724      * locator towards this dpn tep is present or not if the physical locator is
1725      * present go ahead and add the ucast macs otherwise update the mcast mac
1726      * entry to include this dpn tep ip and schedule the job to put ucast macs
1727      * once the physical locator is programmed in device
1728      *
1729      * @param elanName
1730      *            the elan name
1731      * @param lstElanInterfaceNames
1732      *            the lst Elan interface names
1733      * @param dpnId
1734      *            the dpn id
1735      * @param externalNodeId
1736      *            the external node id
1737      */
1738     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1739             NodeId externalNodeId) {
1740         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1741                 externalNodeId.getValue());
1742         if (elanL2GwDevice == null) {
1743             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1744             return;
1745         }
1746         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1747         if (dpnTepIp == null) {
1748             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1749                     externalNodeId);
1750             return;
1751         }
1752
1753         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1754         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1755                 LogicalDatastoreType.OPERATIONAL);
1756         boolean phyLocAlreadyExists =
1757                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1758                 dpnTepIp);
1759         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1760                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1761         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1762
1763         if (phyLocAlreadyExists) {
1764             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1765             return;
1766         }
1767         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1768         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1769     }
1770
1771     /**
1772      * Validate external tunnel state event.
1773      *
1774      * @param externalTunnel
1775      *            the external tunnel
1776      * @param intrf
1777      *            the intrf
1778      * @return true, if successful
1779      */
1780     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1781         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1782             String srcDevice = externalTunnel.getDestinationDevice();
1783             String destDevice = externalTunnel.getSourceDevice();
1784             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1785                     LogicalDatastoreType.CONFIGURATION);
1786             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1787                     otherEndPointExtTunnel);
1788             if (otherEndPointExtTunnel != null) {
1789                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1790                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1791                 if (otherEndPointInterfaceOperational) {
1792                     return true;
1793                 } else {
1794                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1795                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1796                 }
1797             }
1798         }
1799         return false;
1800     }
1801
1802     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1803         List<MatchInfo> mkMatches = new ArrayList<>();
1804         // Matching metadata
1805         mkMatches.add(
1806                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1807         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1808         return mkMatches;
1809     }
1810
1811     @Override
1812     protected ElanInterfaceManager getDataTreeChangeListener() {
1813         return this;
1814     }
1815
1816     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1817                                              Uint64 dpId) {
1818         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1819         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1820             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1821                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1822         try {
1823             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1824         } catch (InterruptedException e) {
1825             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1826         }
1827     }
1828 }