elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 140 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 141 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 142 import org.opendaylight.yangtools.yang.common.Uint32;
 143 import org.opendaylight.yangtools.yang.common.Uint64;
 144 import org.slf4j.Logger;
 145 import org.slf4j.LoggerFactory;
 146
 147 /**
 148  * Class in charge of handling creations, modifications and removals of
 149  * ElanInterfaces.
 150  *
 151  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 152  */
 153 @Singleton
 154 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 155         implements RecoverableListener {
 156     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 157     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 158     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 159     private static final boolean SH_FLAG_SET = true;
 160     private static final boolean SH_FLAG_UNSET = false;
 161
 162     private final DataBroker broker;
 163     private final ManagedNewTransactionRunner txRunner;
 164     private final IMdsalApiManager mdsalManager;
 165     private final IInterfaceManager interfaceManager;
 166     private final IdManagerService idManager;
 167     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 168     private final INeutronVpnManager neutronVpnManager;
 169     private final ElanItmUtils elanItmUtils;
 170     private final ElanEtreeUtils elanEtreeUtils;
 171     private final ElanL2GatewayUtils elanL2GatewayUtils;
 172     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 173     private final ElanUtils elanUtils;
 174     private final JobCoordinator jobCoordinator;
 175     private final ElanInstanceCache elanInstanceCache;
 176     private final ElanInterfaceCache elanInterfaceCache;
 177     private final ElanGroupCache elanGroupCache;
 178
 179     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 180         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 181
 182     @Inject
 183     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 184                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 185                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 186                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 187                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 188                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 189                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 190                                 final ElanInstanceCache elanInstanceCache,
 191                                 final ElanInterfaceCache elanInterfaceCache,
 192                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 193                                 ElanGroupCache elanGroupCache,
 194                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 195         super(ElanInterface.class, ElanInterfaceManager.class);
 196         this.broker = dataBroker;
 197         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 198         this.idManager = managerService;
 199         this.mdsalManager = mdsalApiManager;
 200         this.interfaceManager = interfaceManager;
 201         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 202         this.neutronVpnManager = neutronVpnManager;
 203         this.elanItmUtils = elanItmUtils;
 204         this.elanEtreeUtils = elanEtreeUtils;
 205         this.elanL2GatewayUtils = elanL2GatewayUtils;
 206         this.elanUtils = elanUtils;
 207         this.jobCoordinator = jobCoordinator;
 208         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 209         this.elanInstanceCache = elanInstanceCache;
 210         this.elanInterfaceCache = elanInterfaceCache;
 211         this.elanGroupCache = elanGroupCache;
 212         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 213     }
 214
 215     @Override
 216     @PostConstruct
 217     public void init() {
 218         registerListener();
 219     }
 220
 221     @Override
 222     public void registerListener() {
 223         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 224     }
 225
 226     @Override
 227     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 228         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 229     }
 230
 231     @Override
 232     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 233         String interfaceName = del.getName();
 234         String elanInstanceName = del.getElanInstanceName();
 235         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 236         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 237         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 238             elanInterfaces.remove(del);
 239             if (elanInterfaces.isEmpty()) {
 240                 unProcessedElanInterfaces.remove(elanInstanceName);
 241             }
 242         }
 243         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 244         /*
 245          * Handling in case the elan instance is deleted.If the Elan instance is
 246          * deleted, there is no need to explicitly delete the elan interfaces
 247          */
 248         if (elanInfo == null) {
 249             return;
 250         }
 251         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 252         if (interfaceInfo == null && elanInfo.isExternal()) {
 253             // In deleting external network, the underlying ietf Inteface might have been removed
 254             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 255             // from Operational DS instead
 256             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 257         }
 258         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 259                 interfaceName, interfaceInfo, this);
 260         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 261     }
 262
 263     private static class RemoveElanInterfaceHolder {
 264         boolean isLastElanInterface = false;
 265         Uint64 dpId = null;
 266     }
 267
 268     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 269     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 270             InterfaceInfo interfaceInfo) {
 271         String elanName = elanInfo.getElanInstanceName();
 272         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 273         Uint32 elanTag = elanInfo.getElanTag();
 274         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 275         List<ListenableFuture<Void>> futures = new ArrayList<>();
 276         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 277         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 278             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 279             if (elanState == null) {
 280                 return;
 281             }
 282             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 283                 List<String> elanInterfaces = elanState.getElanInterfaces();
 284                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 285                     holder.isLastElanInterface = true;
 286                 }
 287                 if (interfaceInfo != null) {
 288                     holder.dpId = interfaceInfo.getDpId();
 289                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 290                         interfaceName, elanTag, interfaceTx);
 291                     /*
 292                      * If there are not elan ports, remove the unknown dmac, terminating
 293                      * service table flows, remote/local bc group
 294                      */
 295                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 296                         || dpnInterfaces.getInterfaces().isEmpty()) {
 297                         // No more Elan Interfaces in this DPN
 298                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 299                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 300                             holder.dpId);
 301                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 302                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 303                         }
 304                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 305                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 306                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 307                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 308                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 309                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 310                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 311                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 312                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 313                             }
 314                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 315                         }
 316                     } else {
 317                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 318                     }
 319                 }
 320             }));
 321         }));
 322         futures.forEach(ElanUtils::waitForTransactionToComplete);
 323
 324         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 325                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 326         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 327                 ElanConstants.JOB_MAX_RETRIES);
 328
 329         return futures;
 330     }
 331
 332     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 333             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 334             throws ExecutionException, InterruptedException {
 335         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 336         if (etreeLeafTag != null) {
 337             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 338             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 339         }
 340     }
 341
 342     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 343             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 344             throws ExecutionException, InterruptedException {
 345         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 346         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 347     }
 348
 349     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 350             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 351             throws ExecutionException, InterruptedException {
 352         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 353         if (etreeInstance != null) {
 354             Uint64 dpnId = interfaceInfo.getDpId();
 355             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 356             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 357             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 358         }
 359     }
 360
 361     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 362             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 363             throws ExecutionException, InterruptedException {
 364         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 365         if (etreeInstance != null) {
 366             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 367             Uint64 dpnId = interfaceInfo.getDpId();
 368             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 369             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 370             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 371         }
 372     }
 373
 374     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 375             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 376         String elanName = elanInfo.getElanInstanceName();
 377         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 378         if (elanState == null) {
 379             return elanState;
 380         }
 381         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 382         List<String> elanInterfaces = new ArrayList<>();
 383         if (existingElanInterfaces != null) {
 384             elanInterfaces.addAll(existingElanInterfaces);
 385         }
 386         boolean isRemoved = elanInterfaces.remove(interfaceName);
 387         if (!isRemoved) {
 388             return elanState;
 389         }
 390
 391         if (elanInterfaces.isEmpty()) {
 392             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 393             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 394             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 395             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 396         } else {
 397             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 398                     .withKey(new ElanKey(elanName)).build();
 399             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 400         }
 401         return elanState;
 402     }
 403
 404     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 405             throws ReadFailedException {
 406         // removing the ElanInterface from the config data_store if interface is
 407         // not present in Interface config DS
 408         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 409             interfaceName) == null
 410                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 411             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 412         }
 413     }
 414
 415     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 416             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 417         String elanName = elanInfo.getElanInstanceName();
 418         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 419         List<ListenableFuture<Void>> futures = new ArrayList<>();
 420         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 421             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 422                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 423                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 424                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 425                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 426                 if (interfaceInfo != null) {
 427                     if (existingElanInterfaceMac.isPresent()) {
 428                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 429                         if (existingMacEntries != null) {
 430                             List<PhysAddress> macAddresses = new ArrayList<>();
 431                             for (MacEntry macEntry : existingMacEntries) {
 432                                 PhysAddress macAddress = macEntry.getMacAddress();
 433                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 434                                         macAddress.getValue(), interfaceName);
 435                                 Optional<MacEntry> macEntryOptional =
 436                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 437                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 438                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 439                                 }
 440                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 441                                         interfaceName, elanName, macAddress);
 442                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 443                                 macAddresses.add(macAddress);
 444                             }
 445
 446                             // Removing all those MACs from External Devices belonging
 447                             // to this ELAN
 448                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 449                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 450                             }
 451                         }
 452                     }
 453                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 454                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 455                 } else if (existingElanInterfaceMac.isPresent()) {
 456                     // Interface does not exist in ConfigDS, so lets remove everything
 457                     // about that interface related to Elan
 458                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 459                     if (macEntries != null) {
 460                         for (MacEntry macEntry : macEntries) {
 461                             PhysAddress macAddress = macEntry.getMacAddress();
 462                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 463                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 464                             }
 465                         }
 466                     }
 467                 }
 468                 if (existingElanInterfaceMac.isPresent()) {
 469                     interfaceTx.delete(elanInterfaceId);
 470                 }
 471                 unbindService(interfaceName, flowTx);
 472                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 473             }));
 474         }));
 475         return futures;
 476     }
 477
 478     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 479                                                                          String interfaceName, Uint32 elanTag,
 480                                                                          TypedReadWriteTransaction<Operational> tx)
 481             throws ExecutionException, InterruptedException {
 482         // FIXME: pass in and use ElanInstanceKey instead?
 483         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 484         lock.lock();
 485         try {
 486             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 487             if (dpnInterfaces != null) {
 488                 List<String> interfaceLists = null;
 489                 if (dpnInterfaces.getInterfaces() != null) {
 490                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 491                 }
 492                 if (interfaceLists != null) {
 493                     interfaceLists.remove(interfaceName);
 494                 }
 495
 496                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 497                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 498                     deleteElanDpnInterface(elanName, dpId, tx);
 499                 } else {
 500                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 501                 }
 502             }
 503             return dpnInterfaces;
 504         } finally {
 505             lock.unlock();
 506         }
 507     }
 508
 509     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 510         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 511         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 512             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 513                 Uint64 currentDpId = dpnInterface.getDpId();
 514                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 515                     for (String elanInterface : dpnInterface.getInterfaces()) {
 516                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 517                         if (macs == null || macs.getMacEntry() == null) {
 518                             continue;
 519                         }
 520                         for (MacEntry mac : macs.getMacEntry()) {
 521                             removeTheMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 522                             removeEtreeMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 523                         }
 524                     }
 525                 }
 526             }
 527         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 528     }
 529
 530     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 531             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 532         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 533         if (etreeLeafTag != null) {
 534             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), mac, confTx);
 535         }
 536     }
 537
 538     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 539             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 540         mdsalManager
 541                 .removeFlow(confTx, dpId,
 542                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 543                                 ElanUtils.getKnownDynamicmacFlowRef(elanTag, mac.getMacAddress().getValue())));
 544     }
 545
 546     /*
 547     * Possible Scenarios for update
 548     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 549         then all {1,2,3,4} should be removed
 550
 551         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 552         then all {1,2,3,4} should be added
 553
 554         c. if orig = {1,2,3,4} updated={2,3,4}
 555         then 1 should be removed
 556
 557         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 558         then we should just add 5
 559
 560         e. if orig = {1,2,3,4} updated={2,3,4,5}
 561         then 1 should be removed , 5 should be added
 562     * */
 563     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 564     @Override
 565     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 566         // updating the static-Mac Entries for the existing elanInterface
 567         String elanName = update.getElanInstanceName();
 568         String interfaceName = update.getName();
 569         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 570         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 571
 572         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 573         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 574         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 575         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 576
 577         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 578                 deletedEntry.getMacAddress()));
 579
 580         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 581         * Hence add the macentries for the same.*/
 582         for (StaticMacEntries staticMacEntry : updatedEntries) {
 583             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 584                     staticMacEntry.getMacAddress());
 585             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 586                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 587                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 588                     if (existingMacEntry.isPresent()) {
 589                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 590                                 existingMacEntry.get(), elanName);
 591                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 592                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 593                             tx);
 594                     } else {
 595                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 596                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 597                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 598                             elanName, interfaceName, staticMacEntry, tx);
 599                     }
 600                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 601         }
 602     }
 603
 604     @Override
 605     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 606         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 607         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 608             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 609             String interfaceName = elanInterfaceAdded.getName();
 610             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 611             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 612             if (interfaceInfo == null) {
 613                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 614                 return;
 615             }
 616             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 617
 618             if (elanInstance == null) {
 619                 // Add the ElanInstance in the Configuration data-store
 620                 List<String> elanInterfaces = new ArrayList<>();
 621                 elanInterfaces.add(interfaceName);
 622                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 623                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 624                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 625                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 626             }
 627
 628             Uint32 elanTag = elanInstance.getElanTag();
 629             // If elan tag is not updated, then put the elan interface into
 630             // unprocessed entry map and entry. Let entries
 631             // in this map get processed during ELAN update DCN.
 632             if (elanTag == null || elanTag.longValue() == 0L) {
 633                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 634                 if (elanInterfaces == null) {
 635                     elanInterfaces = new ConcurrentLinkedQueue<>();
 636                 }
 637                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 638                     elanInterfaces.add(elanInterfaceAdded);
 639                 }
 640                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 641                 return;
 642             }
 643             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 644                     interfaceInfo, elanInstance, this);
 645             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 646         }), LOG, "Error processing added ELAN interface");
 647     }
 648
 649     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 650         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 651         List<ListenableFuture<Void>> futures = new ArrayList<>();
 652         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 653         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 654             return futures;
 655         }
 656         for (ElanInterface elanInterface : elanInterfaces) {
 657             String interfaceName = elanInterface.getName();
 658             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 659             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 660         }
 661         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 662         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 663                 elanInstance.getElanInstanceName());
 664         return futures;
 665     }
 666
 667     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 668             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 669         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 670                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 671         List<DpnInterfaces> dpnInterfaceLists = null;
 672         if (elanDpnInterfacesList != null) {
 673             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 674         }
 675         if (dpnInterfaceLists != null && !dpnInterfaceLists.isEmpty()) {
 676             Uint64 dstDpId = interfaceInfo.getDpId();
 677             processRemoteDmacFlowForInterface(dstDpId, elanInstance, dpnInterfaceLists, writeFlowGroupTx);
 678         }
 679     }
 680
 681     private void processRemoteDmacFlowForInterface(Uint64 dstDpId, ElanInstance elanInstance,
 682              List<DpnInterfaces> dpnInterfaceLists, TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 683         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 684             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 685                 continue;
 686             }
 687             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 688             if (remoteElanInterfaces == null || remoteElanInterfaces.isEmpty()) {
 689                 continue;
 690             }
 691             for (String remoteIf : remoteElanInterfaces) {
 692                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 693                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 694                 if (elanIfMac == null || remoteInterface == null) {
 695                     continue;
 696                 }
 697                 List<MacEntry> remoteMacEntries = elanIfMac.nonnullMacEntry();
 698                 for (MacEntry macEntry : remoteMacEntries) {
 699                     String macAddress = macEntry.getMacAddress().getValue();
 700                     LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 701                             dstDpId, elanInstance.getElanInstanceName());
 702                     elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 703                             remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 704                             elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 705                 }
 706             }
 707         }
 708     }
 709
 710     private static class AddElanInterfaceHolder {
 711         private DpnInterfaces dpnInterfaces = null;
 712         private boolean isFirstInterfaceInDpn = false;
 713         private Uint64 dpId;
 714     }
 715
 716     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 717     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 718             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 719         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 720         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 721         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 722
 723         String interfaceName = elanInterface.getName();
 724         String elanInstanceName = elanInterface.getElanInstanceName();
 725         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 726         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 727
 728         List<ListenableFuture<Void>> futures = new ArrayList<>();
 729         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 730         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 731             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 732             if (elanInfo == null) {
 733                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 734                 List<String> elanInterfaces = new ArrayList<>();
 735                 elanInterfaces.add(interfaceName);
 736                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 737                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 738                         operTx)));
 739             } else {
 740                 createElanStateList(elanInstanceName, interfaceName, operTx);
 741             }
 742             // Specific actions to the DPN where the ElanInterface has been added,
 743             // for example, programming the
 744             // External tunnel table if needed or adding the ElanInterface to the
 745             // DpnInterfaces in the operational DS.
 746             holder.dpId = interfaceInfo.getDpId();
 747             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 748                 // FIXME: use elanInstaince.key() instead?
 749                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 750                 lock.lock();
 751                 try {
 752                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 753                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 754                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 755                     if (ElanUtils.isVlan(elanInstance)) {
 756                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 757                             elanInstanceName, existingElanDpnInterfaces);
 758                     } else {
 759                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 760                     }
 761                     if (holder.isFirstInterfaceInDpn) {
 762                         // ELAN's 1st ElanInterface added to this DPN
 763                         if (!existingElanDpnInterfaces.isPresent()) {
 764                             holder.dpnInterfaces =
 765                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 766                         } else {
 767                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 768                             List<String> elanInterfaces =
 769                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 770                             elanInterfaces.add(interfaceName);
 771                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 772                                 elanInterfaces, operTx);
 773                         }
 774                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 775                                 interfaceName, elanInstanceName, holder.dpId);
 776                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 777                         // table, but only if Elan has VNI
 778                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 779                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 780                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 781                         }
 782                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 783                             interfaceName);
 784                     } else {
 785                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 786                         List<String> elanInterfaces =
 787                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 788                         elanInterfaces.add(interfaceName);
 789                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 790                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 791                                 interfaceName);
 792                         }
 793                         holder.dpnInterfaces =
 794                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 795                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 796                                 interfaceName, elanInstanceName, holder.dpId);
 797                     }
 798                 } finally {
 799                     lock.unlock();
 800                 }
 801             }
 802
 803             // add code to install Local/Remote BC group, unknow DMAC entry,
 804             // terminating service table flow entry
 805             // call bindservice of interfacemanager to create ingress table flow
 806             // enty.
 807         }));
 808         futures.forEach(ElanUtils::waitForTransactionToComplete);
 809         futures.add(
 810             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 811                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 812                     holder.isFirstInterfaceInDpn, confTx))));
 813
 814         // add the vlan provider interface to remote BC group for the elan
 815         // for internal vlan networks
 816         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 817             if (interfaceManager.isExternalInterface(interfaceName)) {
 818                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 819                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 820             }
 821         }
 822         if (holder.isFirstInterfaceInDpn) {
 823             // ELAN's 1st ElanInterface added to this DPN
 824             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 825             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 826                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 827                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 828             }));
 829         } else {
 830             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 831         }
 832
 833         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 834                 holder.isFirstInterfaceInDpn, elanInterface);
 835         return futures;
 836     }
 837
 838     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 839     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 840             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 841         String elanInstanceName = elanInstance.getElanInstanceName();
 842         String interfaceName = elanInterface.getName();
 843         List<ListenableFuture<Void>> futures = new ArrayList<>();
 844         Uint64 dpId = interfaceInfo.getDpId();
 845         boolean isInterfaceOperational = isOperational(interfaceInfo);
 846         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 847             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 848                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 849                     isFirstInterfaceInDpn, confTx);
 850
 851                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 852                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 853
 854                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 855                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 856                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 857                             staticMacEntry.getMacAddress());
 858                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 859                             LogicalDatastoreType.OPERATIONAL, macId);
 860                         if (existingMacEntry.isPresent()) {
 861                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 862                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 863                                 existingMacEntry.get(), operTx);
 864                         } else {
 865                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 866                                 interfaceName, staticMacEntry, operTx);
 867                         }
 868
 869                         if (isInterfaceOperational) {
 870                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 871                             // DPNs
 872                             String macAddress = staticMacEntry.getMacAddress().getValue();
 873                             LOG.info(
 874                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 875                                     + " {}",
 876                                 macAddress, elanInstanceName, interfaceName);
 877                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 878                                     interfaceName, elanInstanceName, macAddress);
 879                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 880                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 881                         }
 882                     }
 883
 884                     if (isInterfaceOperational) {
 885                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 886                         // on purpose.
 887                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 888                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 889                         }
 890                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 891                             staticMacAddresses);
 892                     }
 893                 }
 894             }));
 895         }));
 896         futures.forEach(ElanUtils::waitForTransactionToComplete);
 897         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 898             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 899             try {
 900                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 901                 if (neutronPort != null) {
 902                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 903                 }
 904             } catch (IllegalArgumentException ex) {
 905                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 906             }
 907         }
 908         return futures;
 909     }
 910
 911     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 912             PhysAddress physAddress) {
 913         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 914         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 915         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 916                 LogicalDatastoreType.OPERATIONAL, macId);
 917
 918         if (!existingMacEntry.isPresent()) {
 919             return;
 920         }
 921
 922         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 923                 .withKey(new MacEntryKey(physAddress)).build();
 924         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 925                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 926     }
 927
 928     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 929             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 930         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 931         if (!existingElanDpnInterfaces.isPresent()) {
 932             return true;
 933         }
 934         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 935             return false;
 936         }
 937         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 938         int dummyInterfaceCount =  0;
 939         List<String> interfaces = dpnInterfaces.getInterfaces();
 940         if (interfaces == null) {
 941             return true;
 942         }
 943         if (interfaces.contains(routerPortUuid)) {
 944             dummyInterfaceCount++;
 945         }
 946         if (interfaces.contains(elanInstanceName)) {
 947             dummyInterfaceCount++;
 948         }
 949         return interfaces.size() == dummyInterfaceCount;
 950     }
 951
 952     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 953             PhysAddress physAddress) {
 954         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 955                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 956     }
 957
 958     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 959             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 960         if (!isOperational(interfaceInfo)) {
 961             LOG.warn("Interface {} is not operational", elanInterface.getName());
 962             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 963                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 964             return;
 965         }
 966         Uint64 dpId = interfaceInfo.getDpId();
 967         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 968             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 969         }
 970         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 971         if (isFirstInterfaceInDpn) {
 972             // Terminating Service , UnknownDMAC Table.
 973             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 974             // for ELAN Instance is VxLAN
 975             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 976                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 977             }
 978             setupUnknownDMacTable(elanInstance, dpId, confTx);
 979             /*
 980              * Install remote DMAC flow. This is required since this DPN is
 981              * added later to the elan instance and remote DMACs of other
 982              * interfaces in this elan instance are not present in the current
 983              * dpn.
 984              */
 985             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 986                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 987                         elanInstance.getElanInstanceName());
 988                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 989             }
 990         }
 991         // bind the Elan service to the Interface
 992         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 993     }
 994
 995     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 996             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 997         if (!isOperational(interfaceInfo)) {
 998             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
 999                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1000             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1001                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1002             return;
1003         }
1004         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1005                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1006         // LocalBroadcast Group creation with elan-Interfaces
1007         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1008                 dpnInterfaces.getDpId());
1009         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1010         if (isFirstInterfaceInDpn) {
1011             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1012             try {
1013                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1014             } catch (InterruptedException e1) {
1015                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1016             }
1017         }
1018     }
1019
1020     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1021         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1022         return InstanceIdentifier.builder(Nodes.class)
1023                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1024                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1025                 .augmentation(FlowCapableNode.class)
1026                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1027     }
1028
1029     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1030                                                               InterfaceInfo interfaceInfo,
1031                                                               DpnInterfaces dpnInterfaces,
1032                                                               boolean isFirstInterfaceInDpn,
1033                                                               ElanInterface elanInterface) {
1034         if (!isOperational(interfaceInfo)) {
1035             LOG.debug("Interface {} is not operational", elanInterface.getName());
1036             return;
1037         }
1038         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1039         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1040                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1041         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1042         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1043             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1044         });
1045     }
1046
1047     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1048             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1049         int ifTag = interfaceInfo.getInterfaceTag();
1050         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1051                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 11, elanInfo.getElanInstanceName(),
1052                 0, 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1053                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1054                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1055
1056         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1057         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1058                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1059
1060         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1061                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 12, elanInfo.getElanInstanceName(), 0,
1062                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1063                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1064
1065         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1066         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1067                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1068     }
1069
1070     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1071             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1072         int ifTag = interfaceInfo.getInterfaceTag();
1073         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1074             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1075
1076         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1077
1078         Flow flowEntity = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1079             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"));
1080
1081         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1082     }
1083
1084     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1085         List<MatchInfo> mkMatches = new ArrayList<>();
1086         MatchInfo match = new MatchTunnelId(vni);
1087         mkMatches.add(match);
1088         return mkMatches;
1089     }
1090
1091     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1092         List<InstructionInfo> mkInstructions = new ArrayList<>();
1093         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1094         return mkInstructions;
1095     }
1096
1097     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1098         List<MatchInfo> mkMatches = new ArrayList<>();
1099         // Matching metadata
1100         mkMatches.add(new MatchMetadata(
1101                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1102         return mkMatches;
1103     }
1104
1105     /**
1106      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1107      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1108      *
1109      * @param elanTag
1110      *            elanTag to be written in metadata when flow is selected
1111      * @return the instructions ready to be installed in a flow
1112      */
1113     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1114         List<InstructionInfo> mkInstructions = new ArrayList<>();
1115         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1116                 ElanHelper.getElanMetadataMask()));
1117         /* applicable for EXTERNAL_TUNNEL_TABLE only
1118         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1119         */
1120         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1121         return mkInstructions;
1122     }
1123
1124     // Install DMAC entry on dst DPN
1125     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1126             Uint64 dstDpId) {
1127         String interfaceName = interfaceInfo.getInterfaceName();
1128         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1129         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1130             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1131             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1132                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1133                     for (MacEntry macEntry : macEntries) {
1134                         String macAddress = macEntry.getMacAddress().getValue();
1135                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1136                             interfaceName);
1137                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1138                             interfaceInfo.getDpId())) {
1139                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1140                                 macAddress);
1141                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1142                         }
1143                     }
1144                 })));
1145         }
1146         return emptyList();
1147     }
1148
1149     private static void createDropBucket(List<Bucket> listBucket) {
1150         List<Action> actionsInfos = new ArrayList<>();
1151         actionsInfos.add(new ActionDrop().buildAction());
1152         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1153                 MDSALUtil.WATCH_GROUP);
1154         listBucket.add(dropBucket);
1155     }
1156
1157     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1158             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1159         if (!isOperational(interfaceInfo)) {
1160             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1161                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1162             return;
1163         }
1164         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1165         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1166     }
1167
1168     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1169             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1170         List<Bucket> listBucket = new ArrayList<>();
1171         int bucketId = 0;
1172         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1173
1174         List<String> interfaces = new ArrayList<>();
1175         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1176             interfaces = newDpnInterface.getInterfaces();
1177         }
1178         for (String ifName : interfaces) {
1179             // In case if there is a InterfacePort in the cache which is not in
1180             // operational state, skip processing it
1181             InterfaceInfo ifInfo = interfaceManager
1182                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1183             if (!isOperational(ifInfo)) {
1184                 continue;
1185             }
1186
1187             if (!interfaceManager.isExternalInterface(ifName)) {
1188                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1189                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1190                 bucketId++;
1191             }
1192         }
1193
1194         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1195                 MDSALUtil.buildBucketLists(listBucket));
1196         LOG.trace("installing the localBroadCast Group:{}", group);
1197         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1198     }
1199
1200     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1201             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1202         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1203         if (etreeInstance != null) {
1204             List<Bucket> listBucket = new ArrayList<>();
1205             int bucketId = 0;
1206
1207             List<String> interfaces = new ArrayList<>();
1208             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1209                 interfaces = newDpnInterface.getInterfaces();
1210             }
1211             for (String ifName : interfaces) {
1212                 // In case if there is a InterfacePort in the cache which is not
1213                 // in
1214                 // operational state, skip processing it
1215                 InterfaceInfo ifInfo = interfaceManager
1216                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1217                 if (!isOperational(ifInfo)) {
1218                     continue;
1219                 }
1220
1221                 if (!interfaceManager.isExternalInterface(ifName)) {
1222                     // only add root interfaces
1223                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1224                 }
1225             }
1226
1227             if (listBucket.isEmpty()) { // No Buckets
1228                 createDropBucket(listBucket);
1229             }
1230
1231             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1232             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1233             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1234                     MDSALUtil.buildBucketLists(listBucket));
1235             LOG.trace("installing the localBroadCast Group:{}", group);
1236             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1237         }
1238     }
1239
1240     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1241             InterfaceInfo ifInfo) {
1242         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1243         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1244             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1245                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1246             bucketId++;
1247         }
1248         return bucketId;
1249     }
1250
1251     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1252             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1253             throws ExecutionException, InterruptedException {
1254         Uint64 dpnId = interfaceInfo.getDpId();
1255         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1256         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1257         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1258     }
1259
1260     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1261             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1262             throws ExecutionException, InterruptedException {
1263         Uint64 dpnId = interfaceInfo.getDpId();
1264         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1265         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1266         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1267     }
1268
1269     /**
1270      * Installs a flow in the External Tunnel table consisting in translating
1271      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1272      * elanTag that will be used later in the ELANs pipeline.
1273      * @param dpnId the dpn id
1274      */
1275     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1276             TypedWriteTransaction<Configuration> confTx) {
1277         Uint32 elanTag = elanInfo.getElanTag();
1278         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1279                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1280                 elanInfo.getElanInstanceName(), // flowName
1281                 0, // idleTimeout
1282                 0, // hardTimeout
1283                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1284                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1285                 getInstructionsIntOrExtTunnelTable(elanTag));
1286
1287         mdsalManager.addFlow(confTx, flowEntity);
1288     }
1289
1290     /**
1291      * Removes, from External Tunnel table, the flow that translates from VNI to
1292      * elanTag. Important: ensure this method is only called whenever there is
1293      * no other ElanInterface in the specified DPN
1294      *
1295      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1296      */
1297     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1298             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1299         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1300         // last ElanInstance plus
1301         // adding a new one does (almost at the same time) are executed in that
1302         // exact order
1303
1304         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1305         FlowEntity flowEntity = new FlowEntityBuilder()
1306             .setDpnId(dpnId)
1307             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1308             .setFlowId(flowId)
1309             .build();
1310         mdsalManager.removeFlow(confTx, flowEntity);
1311     }
1312
1313     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1314             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1315         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1316         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1317     }
1318
1319     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1320             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1321         List<? extends MatchInfoBase> listMatchInfoBase;
1322         List<InstructionInfo> instructionInfos;
1323         Uint32 serviceId;
1324         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1325             serviceId = elanTag;
1326             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1327             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1328         } else {
1329             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1330             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1331             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1332         }
1333         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1334                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1335                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1336                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1337                 listMatchInfoBase, instructionInfos);
1338         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1339         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1340                 elanInfo.getElanInstanceName(), elanTag);
1341     }
1342
1343     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1344             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1345         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1346         if (etreeInstance != null) {
1347             setupTerminateServiceTable(elanInfo, dpId,
1348                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1349         }
1350     }
1351
1352     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1353             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1354         long elanTag = elanInfo.getElanTag().toJava();
1355         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1356         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1357         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1358     }
1359
1360     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1361             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1362         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1363         if (etreeLeafTag != null) {
1364             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1365             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1366             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1367         }
1368     }
1369
1370     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1371             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1372         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1373                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1374                 5, elanInfo.getElanInstanceName(), 0, 0,
1375                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1376                 getMatchesForElanTag(elanTag, /* SH flag */false),
1377                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1378
1379         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1380         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1381                 dpId, elanInfo.getElanInstanceName(), elanTag);
1382     }
1383
1384     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1385             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1386         // only if ELAN can connect to external network, perform the following
1387
1388         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1389             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1390                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1391                     5, elanInfo.getElanInstanceName(), 0, 0,
1392                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1393                     getMatchesForElanTag(elanTag, /* SH flag */true),
1394                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1395             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1396             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1397                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1398         }
1399     }
1400
1401
1402     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1403             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1404             throws ExecutionException, InterruptedException {
1405         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1406                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1407         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1408
1409         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1410             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1411                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1412                     .build();
1413             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1414         }
1415     }
1416
1417     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1418         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1419     }
1420
1421     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1422             TypedWriteTransaction<Configuration> tx) {
1423         if (isStandardElanService(elanInterface)) {
1424             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1425                     elanInterface.getName(), lportTag, tx);
1426         } else { // Etree service
1427             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1428         }
1429     }
1430
1431     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1432             TypedWriteTransaction<Configuration> tx) {
1433         int instructionKey = 0;
1434         List<Instruction> instructions = new ArrayList<>();
1435         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1436                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1437
1438         List<Action> actions = new ArrayList<>();
1439         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1440                 lportTag).buildAction());
1441         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1442                 elanTag).buildAction());
1443         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1444
1445         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1446                 ++instructionKey));
1447
1448         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1449         BoundServices serviceInfo = ElanUtils.getBoundServices(
1450                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1451                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1452         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1453         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1454                 bindServiceId);
1455         if (!existingElanService.isPresent()) {
1456             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1457             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1458         }
1459     }
1460
1461     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1462             TypedWriteTransaction<Configuration> tx) {
1463         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1464             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1465                     lportTag, tx);
1466         } else {
1467             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1468             if (etreeInstance == null) {
1469                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1470                         elanInterface.getName(), elanInfo.getElanInstanceName());
1471             } else {
1472                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1473                         elanInterface.getName(), lportTag, tx);
1474             }
1475         }
1476     }
1477
1478     private static boolean isStandardElanService(ElanInterface elanInterface) {
1479         return elanInterface.augmentation(EtreeInterface.class) == null;
1480     }
1481
1482     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1483             throws ExecutionException, InterruptedException {
1484         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1485         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1486         if (tx.read(bindServiceId).get().isPresent()) {
1487             tx.delete(bindServiceId);
1488         }
1489     }
1490
1491     private static String getFlowRef(long tableId, long elanTag) {
1492         return String.valueOf(tableId) + elanTag;
1493     }
1494
1495     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1496         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1497                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1498     }
1499
1500     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1501         return String.valueOf(tableId) + elanTag + shFlag;
1502     }
1503
1504     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1505         List<Action> listAction = new ArrayList<>();
1506         int actionKey = 0;
1507         listAction.add(
1508             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1509         actionKey++;
1510         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1511         return listAction;
1512     }
1513
1514     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1515             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1516         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1517                 .withKey(new DpnInterfacesKey(dpId)).build();
1518         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1519                 CREATE_MISSING_PARENTS);
1520         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1521                 interfaceNames);
1522         return dpnInterface;
1523     }
1524
1525     /**
1526      * Delete elan dpn interface from operational DS.
1527      *
1528      * @param elanInstanceName
1529      *            the elan instance name
1530      * @param dpId
1531      *            the dp id
1532      */
1533     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1534             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1535         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1536                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1537         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1538         if (dpnInterfaces.isPresent()) {
1539             tx.delete(dpnInterfacesId);
1540         }
1541     }
1542
1543     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1544             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1545         List<String> interfaceNames = new ArrayList<>();
1546         interfaceNames.add(interfaceName);
1547         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1548                 .withKey(new DpnInterfacesKey(dpId)).build();
1549         return dpnInterface;
1550     }
1551
1552     private static void createElanStateList(String elanInstanceName, String interfaceName,
1553             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1554         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1555         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1556         // Adding new Elan Interface Port to the operational DataStore without
1557         // Static-Mac Entries..
1558         if (elanInterfaceLists.isPresent()) {
1559             List<String> interfaceLists = new ArrayList<>();
1560             interfaceLists.add(interfaceName);
1561             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1562             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1563                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1564             }
1565
1566             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1567                     .withKey(new ElanKey(elanInstanceName)).build();
1568             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1569             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1570                     interfaceLists);
1571         }
1572     }
1573
1574     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1575         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1576     }
1577
1578     @SuppressWarnings("checkstyle:IllegalCatch")
1579     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1580         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1581         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1582                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1583         if (dpnInterfaceLists == null) {
1584             return;
1585         }
1586         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1587         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1588             int cnt = 0;
1589             String elanName = elanDpns.getElanInstanceName();
1590             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1591             if (elanInfo == null) {
1592                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1593                         + "skipping this ELAN for tunnel handling", elanName);
1594                 continue;
1595             }
1596             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1597                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1598                 continue;
1599             }
1600             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1601             if (dpnInterfaces == null) {
1602                 continue;
1603             }
1604             DpnInterfaces dstDpnIf = null;
1605             for (DpnInterfaces dpnIf : dpnInterfaces) {
1606                 Uint64 dpnIfDpId = dpnIf.getDpId();
1607                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1608                     cnt++;
1609                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1610                     cnt++;
1611                     dstDpnIf = dpnIf;
1612                 }
1613             }
1614             if (cnt == 2) {
1615                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1616                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1617                 jobCoordinator.enqueueJob(elanName, () -> {
1618                     // update Remote BC Group
1619                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1620                     try {
1621                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1622                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1623                                 confTx)).get();
1624                     } catch (RuntimeException e) {
1625                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1626                     }
1627                     Set<String> interfaceLists = new HashSet<>();
1628                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1629                     for (String ifName : interfaceLists) {
1630                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1631                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1632                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1633                             if (isOperational(interfaceInfo)) {
1634                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1635                             }
1636                             return emptyList();
1637                         }, ElanConstants.JOB_MAX_RETRIES);
1638                     }
1639                     return emptyList();
1640                 }, ElanConstants.JOB_MAX_RETRIES);
1641             }
1642
1643         }
1644     }
1645
1646     /**
1647      * Handle external tunnel state event.
1648      *
1649      * @param externalTunnel
1650      *            the external tunnel
1651      * @param intrf
1652      *            the interface
1653      */
1654     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1655         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1656             return;
1657         }
1658         // dpId/externalNodeId will be available either in source or destination
1659         // based on the tunnel end point
1660         Uint64 dpId = null;
1661         NodeId externalNodeId = null;
1662         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1663             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1664             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1665         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1666             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1667             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1668         }
1669         if (dpId == null || externalNodeId == null) {
1670             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1671             return;
1672         }
1673
1674         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1675         if (dpnInterfaceLists == null) {
1676             return;
1677         }
1678         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1679         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1680             String elanName = elanDpns.getElanInstanceName();
1681             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1682
1683             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1684             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1685                     || dpnInterfaces.getInterfaces().isEmpty()) {
1686                 continue;
1687             }
1688             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1689
1690             final Uint64 finalDpId = dpId;
1691             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1692                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1693                 "Error setting up ELAN BGs");
1694             // install L2gwDevices local macs in dpn.
1695             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1696             // Install dpn macs on external device
1697             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1698                     externalNodeId);
1699         }
1700         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1701     }
1702
1703     /**
1704      * Installs dpn macs in external device. first it checks if the physical
1705      * locator towards this dpn tep is present or not if the physical locator is
1706      * present go ahead and add the ucast macs otherwise update the mcast mac
1707      * entry to include this dpn tep ip and schedule the job to put ucast macs
1708      * once the physical locator is programmed in device
1709      *
1710      * @param elanName
1711      *            the elan name
1712      * @param lstElanInterfaceNames
1713      *            the lst Elan interface names
1714      * @param dpnId
1715      *            the dpn id
1716      * @param externalNodeId
1717      *            the external node id
1718      */
1719     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1720             NodeId externalNodeId) {
1721         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1722                 externalNodeId.getValue());
1723         if (elanL2GwDevice == null) {
1724             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1725             return;
1726         }
1727         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1728         if (dpnTepIp == null) {
1729             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1730                     externalNodeId);
1731             return;
1732         }
1733
1734         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1735         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1736                 LogicalDatastoreType.OPERATIONAL);
1737         boolean phyLocAlreadyExists =
1738                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1739                 dpnTepIp);
1740         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1741                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1742         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1743
1744         if (phyLocAlreadyExists) {
1745             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1746             return;
1747         }
1748         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1749         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1750     }
1751
1752     /**
1753      * Validate external tunnel state event.
1754      *
1755      * @param externalTunnel
1756      *            the external tunnel
1757      * @param intrf
1758      *            the intrf
1759      * @return true, if successful
1760      */
1761     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1762         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1763             String srcDevice = externalTunnel.getDestinationDevice();
1764             String destDevice = externalTunnel.getSourceDevice();
1765             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1766                     LogicalDatastoreType.CONFIGURATION);
1767             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1768                     otherEndPointExtTunnel);
1769             if (otherEndPointExtTunnel != null) {
1770                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1771                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1772                 if (otherEndPointInterfaceOperational) {
1773                     return true;
1774                 } else {
1775                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1776                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1777                 }
1778             }
1779         }
1780         return false;
1781     }
1782
1783     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1784         List<MatchInfo> mkMatches = new ArrayList<>();
1785         // Matching metadata
1786         mkMatches.add(
1787                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1788         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1789         return mkMatches;
1790     }
1791
1792     @Override
1793     protected ElanInterfaceManager getDataTreeChangeListener() {
1794         return this;
1795     }
1796
1797     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1798                                              Uint64 dpId) {
1799         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1800         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1801             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1802                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1803         try {
1804             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1805         } catch (InterruptedException e) {
1806             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1807         }
1808     }
1809 }