elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FluentFuture;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Optional;
  29 import java.util.Queue;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.locks.ReentrantLock;
  35 import javax.annotation.PreDestroy;
  36 import javax.inject.Inject;
  37 import javax.inject.Singleton;
  38 import org.apache.commons.lang3.StringUtils;
  39 import org.opendaylight.genius.infra.Datastore.Configuration;
  40 import org.opendaylight.genius.infra.Datastore.Operational;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  44 import org.opendaylight.genius.infra.TypedWriteTransaction;
  45 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  46 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  47 import org.opendaylight.genius.itm.globals.ITMConstants;
  48 import org.opendaylight.genius.mdsalutil.FlowEntity;
  49 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  54 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  55 import org.opendaylight.genius.mdsalutil.NwConstants;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  64 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  67 import org.opendaylight.genius.utils.JvmGlobalLocks;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  70 import org.opendaylight.infrautils.utils.concurrent.Executors;
  71 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  72 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  73 import org.opendaylight.mdsal.binding.api.DataBroker;
  74 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesListKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntriesKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 143 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 144 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 145 import org.opendaylight.yangtools.yang.common.Uint32;
 146 import org.opendaylight.yangtools.yang.common.Uint64;
 147 import org.slf4j.Logger;
 148 import org.slf4j.LoggerFactory;
 149
 150 /**
 151  * Class in charge of handling creations, modifications and removals of
 152  * ElanInterfaces.
 153  *
 154  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 155  */
 156 @Singleton
 157 public class ElanInterfaceManager extends AbstractAsyncDataTreeChangeListener<ElanInterface>
 158         implements RecoverableListener {
 159     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 160     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 161     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 162     private static final boolean SH_FLAG_SET = true;
 163     private static final boolean SH_FLAG_UNSET = false;
 164
 165     private final DataBroker broker;
 166     private final ManagedNewTransactionRunner txRunner;
 167     private final IMdsalApiManager mdsalManager;
 168     private final IInterfaceManager interfaceManager;
 169     private final IdManagerService idManager;
 170     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 171     private final INeutronVpnManager neutronVpnManager;
 172     private final ElanItmUtils elanItmUtils;
 173     private final ElanEtreeUtils elanEtreeUtils;
 174     private final ElanL2GatewayUtils elanL2GatewayUtils;
 175     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 176     private final ElanUtils elanUtils;
 177     private final JobCoordinator jobCoordinator;
 178     private final ElanInstanceCache elanInstanceCache;
 179     private final ElanInterfaceCache elanInterfaceCache;
 180     private final ElanGroupCache elanGroupCache;
 181
 182     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 183         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 184
 185     @Inject
 186     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 187                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 188                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 189                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 190                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 191                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 192                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 193                                 final ElanInstanceCache elanInstanceCache,
 194                                 final ElanInterfaceCache elanInterfaceCache,
 195                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 196                                 ElanGroupCache elanGroupCache,
 197                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 198         super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ElanInterfaces.class)
 199                 .child(ElanInterface.class),
 200                 Executors.newListeningSingleThreadExecutor("ElanInterfaceManager", LOG));
 201         this.broker = dataBroker;
 202         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 203         this.idManager = managerService;
 204         this.mdsalManager = mdsalApiManager;
 205         this.interfaceManager = interfaceManager;
 206         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 207         this.neutronVpnManager = neutronVpnManager;
 208         this.elanItmUtils = elanItmUtils;
 209         this.elanEtreeUtils = elanEtreeUtils;
 210         this.elanL2GatewayUtils = elanL2GatewayUtils;
 211         this.elanUtils = elanUtils;
 212         this.jobCoordinator = jobCoordinator;
 213         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 214         this.elanInstanceCache = elanInstanceCache;
 215         this.elanInterfaceCache = elanInterfaceCache;
 216         this.elanGroupCache = elanGroupCache;
 217         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 218     }
 219
 220     public void init() {
 221         LOG.info("{} registered", getClass().getSimpleName());
 222     }
 223
 224     @Override
 225     public void registerListener() {
 226         super.register();
 227         LOG.info("Registering ElanInterfaceManager");
 228     }
 229
 230     @Override
 231     public void deregisterListener() {
 232         super.close();
 233         LOG.info("Deregistering ElanInterfaceManager");
 234     }
 235
 236     @Override
 237     @PreDestroy
 238     public void close() {
 239         super.close();
 240         Executors.shutdownAndAwaitTermination(getExecutorService());
 241     }
 242
 243     @Override
 244     public void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 245         String interfaceName = del.getName();
 246         String elanInstanceName = del.getElanInstanceName();
 247         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 248         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 249         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 250             elanInterfaces.remove(del);
 251             if (elanInterfaces.isEmpty()) {
 252                 unProcessedElanInterfaces.remove(elanInstanceName);
 253             }
 254         }
 255         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orElse(null);
 256         /*
 257          * Handling in case the elan instance is deleted.If the Elan instance is
 258          * deleted, there is no need to explicitly delete the elan interfaces
 259          */
 260         if (elanInfo == null) {
 261             return;
 262         }
 263         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 264         if (interfaceInfo == null && elanInfo.isExternal()) {
 265             // In deleting external network, the underlying ietf Inteface might have been removed
 266             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 267             // from Operational DS instead
 268             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 269         }
 270         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 271                 interfaceName, interfaceInfo, this);
 272         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 273     }
 274
 275     private static class RemoveElanInterfaceHolder {
 276         boolean isLastElanInterface = false;
 277         Uint64 dpId = null;
 278     }
 279
 280     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 281     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 282             InterfaceInfo interfaceInfo) {
 283         String elanName = elanInfo.getElanInstanceName();
 284         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 285         Uint32 elanTag = elanInfo.getElanTag();
 286         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 287         List<ListenableFuture<Void>> futures = new ArrayList<>();
 288         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 289         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 290             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 291             if (elanState == null) {
 292                 return;
 293             }
 294             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 295                 List<String> elanInterfaces = elanState.getElanInterfaces();
 296                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 297                     holder.isLastElanInterface = true;
 298                 }
 299                 if (interfaceInfo != null) {
 300                     holder.dpId = interfaceInfo.getDpId();
 301                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 302                         interfaceName, elanTag, interfaceTx);
 303                     /*
 304                      * If there are not elan ports, remove the unknown dmac, terminating
 305                      * service table flows, remote/local bc group
 306                      */
 307                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 308                         || dpnInterfaces.getInterfaces().isEmpty()) {
 309                         // No more Elan Interfaces in this DPN
 310                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 311                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 312                             holder.dpId);
 313                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 314                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 315                         }
 316                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 317                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 318                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 319                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 320                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 321                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 322                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 323                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 324                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 325                             }
 326                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 327                         }
 328                     } else {
 329                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 330                     }
 331                 }
 332             }));
 333         }));
 334         futures.forEach(ElanUtils::waitForTransactionToComplete);
 335
 336         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 337                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 338         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 339                 ElanConstants.JOB_MAX_RETRIES);
 340
 341         return futures;
 342     }
 343
 344     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 345             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 346             throws ExecutionException, InterruptedException {
 347         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 348         if (etreeLeafTag != null) {
 349             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 350             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 351         }
 352     }
 353
 354     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 358         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 359     }
 360
 361     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 362             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 363             throws ExecutionException, InterruptedException {
 364         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 365         if (etreeInstance != null) {
 366             Uint64 dpnId = interfaceInfo.getDpId();
 367             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 368             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 369             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 370         }
 371     }
 372
 373     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 374             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 375             throws ExecutionException, InterruptedException {
 376         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 377         if (etreeInstance != null) {
 378             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 379             Uint64 dpnId = interfaceInfo.getDpId();
 380             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 381             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 382             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 383         }
 384     }
 385
 386     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 387             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 388         String elanName = elanInfo.getElanInstanceName();
 389         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 390         if (elanState == null) {
 391             return elanState;
 392         }
 393         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 394         List<String> elanInterfaces = new ArrayList<>();
 395         if (existingElanInterfaces != null) {
 396             elanInterfaces.addAll(existingElanInterfaces);
 397         }
 398         boolean isRemoved = elanInterfaces.remove(interfaceName);
 399         if (!isRemoved) {
 400             return elanState;
 401         }
 402
 403         if (elanInterfaces.isEmpty()) {
 404             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 405             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 406             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 407             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 408         } else {
 409             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 410                     .withKey(new ElanKey(elanName)).build();
 411             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 412         }
 413         return elanState;
 414     }
 415
 416     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 417             throws ExecutionException, InterruptedException {
 418         // removing the ElanInterface from the config data_store if interface is
 419         // not present in Interface config DS
 420         InstanceIdentifier<ElanInterface> elanInterfaceId = ElanUtils
 421                 .getElanInterfaceConfigurationDataPathId(interfaceName);
 422         FluentFuture<Optional<ElanInterface>> interfaceOptional = tx.read(elanInterfaceId);
 423         if (!interfaceOptional.get().isPresent() && elanInterfaceCache.get(interfaceName).isPresent()) {
 424             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 425         }
 426     }
 427
 428     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 429             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 430         String elanName = elanInfo.getElanInstanceName();
 431         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 432         List<ListenableFuture<Void>> futures = new ArrayList<>();
 433         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 434             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 435                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 436                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 437                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 438                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 439                 if (interfaceInfo != null) {
 440                     if (existingElanInterfaceMac.isPresent()) {
 441                         Map<MacEntryKey, MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 442                         if (existingMacEntries != null) {
 443                             List<PhysAddress> macAddresses = new ArrayList<>();
 444                             for (MacEntry macEntry : existingMacEntries.values()) {
 445                                 PhysAddress macAddress = macEntry.getMacAddress();
 446                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 447                                         macAddress.getValue(), interfaceName);
 448                                 Optional<MacEntry> macEntryOptional =
 449                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 450                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 451                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 452                                 }
 453                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 454                                         interfaceName, elanName, macAddress);
 455                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 456                                 macAddresses.add(macAddress);
 457                             }
 458
 459                             // Removing all those MACs from External Devices belonging
 460                             // to this ELAN
 461                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 462                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 463                             }
 464                         }
 465                     }
 466                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 467                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 468                 } else if (existingElanInterfaceMac.isPresent()) {
 469                     // Interface does not exist in ConfigDS, so lets remove everything
 470                     // about that interface related to Elan
 471                     Map<MacEntryKey, MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 472                     if (macEntries != null) {
 473                         for (MacEntry macEntry : macEntries.values()) {
 474                             PhysAddress macAddress = macEntry.getMacAddress();
 475                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 476                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 477                             }
 478                         }
 479                     }
 480                 }
 481                 if (existingElanInterfaceMac.isPresent()) {
 482                     interfaceTx.delete(elanInterfaceId);
 483                 }
 484                 unbindService(interfaceName, flowTx);
 485                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 486             }));
 487         }));
 488         return futures;
 489     }
 490
 491     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 492                                                                          String interfaceName, Uint32 elanTag,
 493                                                                          TypedReadWriteTransaction<Operational> tx)
 494             throws ExecutionException, InterruptedException {
 495         // FIXME: pass in and use ElanInstanceKey instead?
 496         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 497         lock.lock();
 498         try {
 499             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 500             if (dpnInterfaces != null) {
 501                 List<String> interfaceLists = null;
 502                 if (dpnInterfaces.getInterfaces() != null) {
 503                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 504                 }
 505                 if (interfaceLists != null) {
 506                     interfaceLists.remove(interfaceName);
 507                 }
 508
 509                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 510                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 511                     deleteElanDpnInterface(elanName, dpId, tx);
 512                 } else {
 513                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 514                 }
 515             }
 516             return dpnInterfaces;
 517         } finally {
 518             lock.unlock();
 519         }
 520     }
 521
 522     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 523         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 524         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 525             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 526                 Uint64 currentDpId = dpnInterface.getDpId();
 527                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 528                     for (String elanInterface : dpnInterface.getInterfaces()) {
 529                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 530                         if (macs == null || macs.getMacEntry() == null) {
 531                             continue;
 532                         }
 533                         for (MacEntry mac : new ArrayList<MacEntry>(macs.getMacEntry().values())) {
 534                             removeTheMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 535                             removeEtreeMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 536                         }
 537                     }
 538                 }
 539             }
 540         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 541     }
 542
 543     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 544             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 545         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 546         if (etreeLeafTag != null) {
 547             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), mac, confTx);
 548         }
 549     }
 550
 551     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 552             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 553         mdsalManager
 554                 .removeFlow(confTx, dpId,
 555                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 556                                 ElanUtils.getKnownDynamicmacFlowRef(elanTag, mac.getMacAddress().getValue())));
 557     }
 558
 559     /*
 560     * Possible Scenarios for update
 561     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 562         then all {1,2,3,4} should be removed
 563
 564         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 565         then all {1,2,3,4} should be added
 566
 567         c. if orig = {1,2,3,4} updated={2,3,4}
 568         then 1 should be removed
 569
 570         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 571         then we should just add 5
 572
 573         e. if orig = {1,2,3,4} updated={2,3,4,5}
 574         then 1 should be removed , 5 should be added
 575     * */
 576     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 577     @Override
 578     public void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 579         // updating the static-Mac Entries for the existing elanInterface
 580         String elanName = update.getElanInstanceName();
 581         String interfaceName = update.getName();
 582         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 583         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 584
 585         List<StaticMacEntries> originalStaticMacEntries = new ArrayList<StaticMacEntries>(original
 586                 .getStaticMacEntries().values());
 587         List<StaticMacEntries> updatedStaticMacEntries = new ArrayList<StaticMacEntries>(update
 588                 .getStaticMacEntries().values());
 589         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 590         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 591
 592         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 593                 deletedEntry.getMacAddress()));
 594
 595         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 596         * Hence add the macentries for the same.*/
 597         for (StaticMacEntries staticMacEntry : updatedEntries) {
 598             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 599                     staticMacEntry.getMacAddress());
 600             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 601                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 602                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 603                     if (existingMacEntry.isPresent()) {
 604                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 605                                 existingMacEntry.get(), elanName);
 606                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 607                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 608                             tx);
 609                     } else {
 610                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 611                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 612                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 613                             elanName, interfaceName, staticMacEntry, tx);
 614                     }
 615                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 616         }
 617     }
 618
 619     @Override
 620     public void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 621         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 622         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 623             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 624             String interfaceName = elanInterfaceAdded.getName();
 625             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 626             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 627             if (interfaceInfo == null) {
 628                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 629                 return;
 630             }
 631             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orElse(null);
 632
 633             if (elanInstance == null) {
 634                 // Add the ElanInstance in the Configuration data-store
 635                 List<String> elanInterfaces = new ArrayList<>();
 636                 elanInterfaces.add(interfaceName);
 637                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 638                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 639                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 640                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 641             }
 642
 643             Uint32 elanTag = elanInstance.getElanTag();
 644             // If elan tag is not updated, then put the elan interface into
 645             // unprocessed entry map and entry. Let entries
 646             // in this map get processed during ELAN update DCN.
 647             if (elanTag == null || elanTag.longValue() == 0L) {
 648                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 649                 if (elanInterfaces == null) {
 650                     elanInterfaces = new ConcurrentLinkedQueue<>();
 651                 }
 652                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 653                     elanInterfaces.add(elanInterfaceAdded);
 654                 }
 655                 LOG.error("ELAN tag for instance {} is not created. Adding it to unprocessed list."
 656                         + " Recreate the network if this message is seen multiple times", elanInstanceName);
 657                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 658                 return;
 659             }
 660             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 661                     interfaceInfo, elanInstance, this);
 662             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 663         }), LOG, "Error processing added ELAN interface");
 664     }
 665
 666     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 667         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 668         List<ListenableFuture<Void>> futures = new ArrayList<>();
 669         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 670         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 671             return futures;
 672         }
 673         for (ElanInterface elanInterface : elanInterfaces) {
 674             String interfaceName = elanInterface.getName();
 675             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 676             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 677         }
 678         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 679         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 680                 elanInstance.getElanInstanceName());
 681         return futures;
 682     }
 683
 684     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 685             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 686         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 687                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 688         List<DpnInterfaces> dpnInterfaceLists = null;
 689         if (elanDpnInterfacesList != null) {
 690             dpnInterfaceLists = new ArrayList<DpnInterfaces>(elanDpnInterfacesList.getDpnInterfaces().values());
 691         }
 692         if (dpnInterfaceLists != null && !dpnInterfaceLists.isEmpty()) {
 693             Uint64 dstDpId = interfaceInfo.getDpId();
 694             processRemoteDmacFlowForInterface(dstDpId, elanInstance, dpnInterfaceLists, writeFlowGroupTx);
 695         }
 696     }
 697
 698     private void processRemoteDmacFlowForInterface(Uint64 dstDpId, ElanInstance elanInstance,
 699              List<DpnInterfaces> dpnInterfaceLists, TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 700         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 701             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 702                 continue;
 703             }
 704             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 705             if (remoteElanInterfaces == null || remoteElanInterfaces.isEmpty()) {
 706                 continue;
 707             }
 708             for (String remoteIf : remoteElanInterfaces) {
 709                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 710                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 711                 if (elanIfMac == null || remoteInterface == null) {
 712                     continue;
 713                 }
 714                 Map<MacEntryKey, MacEntry> remoteMacEntries = elanIfMac.nonnullMacEntry();
 715                 for (MacEntry macEntry : remoteMacEntries.values()) {
 716                     String macAddress = macEntry.getMacAddress().getValue();
 717                     LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 718                             dstDpId, elanInstance.getElanInstanceName());
 719                     elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 720                             remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 721                             elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 722                 }
 723             }
 724         }
 725     }
 726
 727     private static class AddElanInterfaceHolder {
 728         private DpnInterfaces dpnInterfaces = null;
 729         private boolean isFirstInterfaceInDpn = false;
 730         private Uint64 dpId;
 731     }
 732
 733     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 734     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 735             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 736         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 737         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 738         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 739
 740         String interfaceName = elanInterface.getName();
 741         String elanInstanceName = elanInterface.getElanInstanceName();
 742         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 743         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 744
 745         List<ListenableFuture<Void>> futures = new ArrayList<>();
 746         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 747         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 748             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 749             if (elanInfo == null) {
 750                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 751                 List<String> elanInterfaces = new ArrayList<>();
 752                 elanInterfaces.add(interfaceName);
 753                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 754                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 755                         operTx)));
 756             } else {
 757                 createElanStateList(elanInstanceName, interfaceName, operTx);
 758             }
 759             // Specific actions to the DPN where the ElanInterface has been added,
 760             // for example, programming the
 761             // External tunnel table if needed or adding the ElanInterface to the
 762             // DpnInterfaces in the operational DS.
 763             holder.dpId = interfaceInfo.getDpId();
 764             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 765                 // FIXME: use elanInstaince.key() instead?
 766                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 767                 lock.lock();
 768                 try {
 769                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 770                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 771                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 772                     if (ElanUtils.isVlan(elanInstance)) {
 773                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 774                             elanInstanceName, existingElanDpnInterfaces);
 775                     } else {
 776                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 777                     }
 778                     if (holder.isFirstInterfaceInDpn) {
 779                         // ELAN's 1st ElanInterface added to this DPN
 780                         if (!existingElanDpnInterfaces.isPresent()) {
 781                             holder.dpnInterfaces =
 782                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 783                         } else {
 784                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 785                             List<String> elanInterfaces =
 786                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 787                             elanInterfaces.add(interfaceName);
 788                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 789                                 elanInterfaces, operTx);
 790                         }
 791                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 792                                 interfaceName, elanInstanceName, holder.dpId);
 793                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 794                         // table, but only if Elan has VNI
 795                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 796                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 797                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 798                         }
 799                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 800                             interfaceName);
 801                     } else {
 802                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 803                         List<String> elanInterfaces =
 804                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 805                         elanInterfaces.add(interfaceName);
 806                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 807                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 808                                 interfaceName);
 809                         }
 810                         holder.dpnInterfaces =
 811                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 812                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 813                                 interfaceName, elanInstanceName, holder.dpId);
 814                     }
 815                 } finally {
 816                     lock.unlock();
 817                 }
 818             }
 819
 820             // add code to install Local/Remote BC group, unknow DMAC entry,
 821             // terminating service table flow entry
 822             // call bindservice of interfacemanager to create ingress table flow
 823             // enty.
 824         }));
 825         futures.forEach(ElanUtils::waitForTransactionToComplete);
 826         futures.add(
 827             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 828                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 829                     holder.isFirstInterfaceInDpn, confTx))));
 830
 831         // add the vlan provider interface to remote BC group for the elan
 832         // for internal vlan networks
 833         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 834             if (interfaceManager.isExternalInterface(interfaceName)) {
 835                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 836                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 837             }
 838         }
 839         if (holder.isFirstInterfaceInDpn) {
 840             // ELAN's 1st ElanInterface added to this DPN
 841             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 842             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 843                 operTx.mergeParentStructurePut(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName,
 844                         holder.dpId), holder.dpnInterfaces);
 845             }));
 846         } else {
 847             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 848         }
 849
 850         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 851                 holder.isFirstInterfaceInDpn, elanInterface);
 852         return futures;
 853     }
 854
 855     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 856     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 857             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 858         String elanInstanceName = elanInstance.getElanInstanceName();
 859         String interfaceName = elanInterface.getName();
 860         List<ListenableFuture<Void>> futures = new ArrayList<>();
 861         Uint64 dpId = interfaceInfo.getDpId();
 862         boolean isInterfaceOperational = isOperational(interfaceInfo);
 863         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 864             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 865                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 866                     isFirstInterfaceInDpn, confTx);
 867
 868                 Map<StaticMacEntriesKey, StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 869                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 870
 871                 if (ElanUtils.isNotEmpty(staticMacEntriesList.values())) {
 872                     for (StaticMacEntries staticMacEntry : staticMacEntriesList.values()) {
 873                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 874                             staticMacEntry.getMacAddress());
 875                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 876                             LogicalDatastoreType.OPERATIONAL, macId);
 877                         if (existingMacEntry.isPresent()) {
 878                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 879                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 880                                 existingMacEntry.get(), operTx);
 881                         } else {
 882                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 883                                 interfaceName, staticMacEntry, operTx);
 884                         }
 885
 886                         if (isInterfaceOperational) {
 887                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 888                             // DPNs
 889                             String macAddress = staticMacEntry.getMacAddress().getValue();
 890                             LOG.info(
 891                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 892                                     + " {}",
 893                                 macAddress, elanInstanceName, interfaceName);
 894                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 895                                     interfaceName, elanInstanceName, macAddress);
 896                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 897                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 898                         }
 899                     }
 900
 901                     if (isInterfaceOperational) {
 902                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 903                         // on purpose.
 904                         for (StaticMacEntries staticMacEntry : staticMacEntriesList.values()) {
 905                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 906                         }
 907                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 908                             staticMacAddresses);
 909                     }
 910                 }
 911             }));
 912         }));
 913         futures.forEach(ElanUtils::waitForTransactionToComplete);
 914         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 915             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 916             try {
 917                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 918                 if (neutronPort != null) {
 919                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 920                 }
 921             } catch (IllegalArgumentException ex) {
 922                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 923             }
 924         }
 925         return futures;
 926     }
 927
 928     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 929             PhysAddress physAddress) {
 930         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 931         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 932         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 933                 LogicalDatastoreType.OPERATIONAL, macId);
 934
 935         if (!existingMacEntry.isPresent()) {
 936             return;
 937         }
 938
 939         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 940                 .withKey(new MacEntryKey(physAddress)).build();
 941         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 942                 elanInstanceCache.get(elanInstanceName).orElse(null), interfaceInfo, macEntry);
 943     }
 944
 945     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 946             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 947         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 948         if (!existingElanDpnInterfaces.isPresent()) {
 949             return true;
 950         }
 951         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 952             return false;
 953         }
 954         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 955         int dummyInterfaceCount =  0;
 956         List<String> interfaces = dpnInterfaces.getInterfaces();
 957         if (interfaces == null) {
 958             return true;
 959         }
 960         if (interfaces.contains(routerPortUuid)) {
 961             dummyInterfaceCount++;
 962         }
 963         if (interfaces.contains(elanInstanceName)) {
 964             dummyInterfaceCount++;
 965         }
 966         return interfaces.size() == dummyInterfaceCount;
 967     }
 968
 969     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 970             PhysAddress physAddress) {
 971         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 972                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 973     }
 974
 975     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 976             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 977         if (!isOperational(interfaceInfo)) {
 978             LOG.warn("Interface {} is not operational", elanInterface.getName());
 979             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 980                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 981             return;
 982         }
 983         Uint64 dpId = interfaceInfo.getDpId();
 984         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 985             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 986         }
 987         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 988         if (isFirstInterfaceInDpn) {
 989             // Terminating Service , UnknownDMAC Table.
 990             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 991             // for ELAN Instance is VxLAN
 992             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 993                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 994             }
 995             setupUnknownDMacTable(elanInstance, dpId, confTx);
 996             /*
 997              * Install remote DMAC flow. This is required since this DPN is
 998              * added later to the elan instance and remote DMACs of other
 999              * interfaces in this elan instance are not present in the current
1000              * dpn.
1001              */
1002             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
1003                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
1004                         elanInstance.getElanInstanceName());
1005                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
1006             }
1007         }
1008         // bind the Elan service to the Interface
1009         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
1010     }
1011
1012     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1013             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
1014         if (!isOperational(interfaceInfo)) {
1015             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
1016                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1017             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1018                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1019             return;
1020         }
1021         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1022                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1023         // LocalBroadcast Group creation with elan-Interfaces
1024         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1025                 dpnInterfaces.getDpId());
1026         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1027         if (isFirstInterfaceInDpn) {
1028             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1029             try {
1030                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1031             } catch (InterruptedException e1) {
1032                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1033             }
1034         }
1035     }
1036
1037     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1038         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1039         return InstanceIdentifier.builder(Nodes.class)
1040                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1041                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1042                 .augmentation(FlowCapableNode.class)
1043                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1044     }
1045
1046     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1047                                                               InterfaceInfo interfaceInfo,
1048                                                               DpnInterfaces dpnInterfaces,
1049                                                               boolean isFirstInterfaceInDpn,
1050                                                               ElanInterface elanInterface) {
1051         if (!isOperational(interfaceInfo)) {
1052             LOG.debug("Interface {} is not operational", elanInterface.getName());
1053             return;
1054         }
1055         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1056         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1057                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1058         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1059         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1060             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1061         });
1062     }
1063
1064     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1065             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1066         int ifTag = interfaceInfo.getInterfaceTag();
1067         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1068                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 11, elanInfo.getElanInstanceName(),
1069                 0, 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1070                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1071                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1072
1073         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1074         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1075                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1076         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<InstructionKey, Instruction>();
1077         int instructionKey = 0;
1078         for (Instruction instructionObj : MDSALUtil.buildInstructionsDrop()) {
1079             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
1080         }
1081
1082         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1083                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 12, elanInfo.getElanInstanceName(), 0,
1084                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1085                 getMatchesForFilterEqualsLPortTag(ifTag), customInstructionsMap);
1086
1087         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1088         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1089                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1090     }
1091
1092     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1093             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1094         int ifTag = interfaceInfo.getInterfaceTag();
1095         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1096             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1097
1098         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1099
1100         Flow flowEntity = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1101             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"));
1102
1103         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1104     }
1105
1106     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1107         List<MatchInfo> mkMatches = new ArrayList<>();
1108         MatchInfo match = new MatchTunnelId(vni);
1109         mkMatches.add(match);
1110         return mkMatches;
1111     }
1112
1113     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1114         List<InstructionInfo> mkInstructions = new ArrayList<>();
1115         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1116         return mkInstructions;
1117     }
1118
1119     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1120         List<MatchInfo> mkMatches = new ArrayList<>();
1121         // Matching metadata
1122         mkMatches.add(new MatchMetadata(
1123                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1124         return mkMatches;
1125     }
1126
1127     /**
1128      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1129      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1130      *
1131      * @param elanTag
1132      *            elanTag to be written in metadata when flow is selected
1133      * @return the instructions ready to be installed in a flow
1134      */
1135     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1136         List<InstructionInfo> mkInstructions = new ArrayList<>();
1137         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1138                 ElanHelper.getElanMetadataMask()));
1139         /* applicable for EXTERNAL_TUNNEL_TABLE only
1140         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1141         */
1142         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1143         return mkInstructions;
1144     }
1145
1146     // Install DMAC entry on dst DPN
1147     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1148             Uint64 dstDpId) {
1149         String interfaceName = interfaceInfo.getInterfaceName();
1150         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1151         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1152             Map<MacEntryKey, MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1153             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1154                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1155                     for (MacEntry macEntry : macEntries.values()) {
1156                         String macAddress = macEntry.getMacAddress().getValue();
1157                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1158                             interfaceName);
1159                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1160                             interfaceInfo.getDpId())) {
1161                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1162                                 macAddress);
1163                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1164                         }
1165                     }
1166                 })));
1167         }
1168         return emptyList();
1169     }
1170
1171     private static void createDropBucket(List<Bucket> listBucket) {
1172         List<Action> actionsInfos = new ArrayList<>();
1173         actionsInfos.add(new ActionDrop().buildAction());
1174         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1175                 MDSALUtil.WATCH_GROUP);
1176         listBucket.add(dropBucket);
1177     }
1178
1179     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1180             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1181         if (!isOperational(interfaceInfo)) {
1182             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1183                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1184             return;
1185         }
1186         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1187         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1188     }
1189
1190     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1191             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1192         List<Bucket> listBucket = new ArrayList<>();
1193         int bucketId = 0;
1194         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1195
1196         List<String> interfaces = new ArrayList<>();
1197         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1198             interfaces = newDpnInterface.getInterfaces();
1199         }
1200         for (String ifName : interfaces) {
1201             // In case if there is a InterfacePort in the cache which is not in
1202             // operational state, skip processing it
1203             InterfaceInfo ifInfo = interfaceManager
1204                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1205             if (!isOperational(ifInfo)) {
1206                 continue;
1207             }
1208
1209             if (!interfaceManager.isExternalInterface(ifName)) {
1210                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1211                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1212                 bucketId++;
1213             }
1214         }
1215
1216         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1217                 MDSALUtil.buildBucketLists(listBucket));
1218         LOG.trace("installing the localBroadCast Group:{}", group);
1219         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1220     }
1221
1222     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1223             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1224         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1225         if (etreeInstance != null) {
1226             List<Bucket> listBucket = new ArrayList<>();
1227             int bucketId = 0;
1228
1229             List<String> interfaces = new ArrayList<>();
1230             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1231                 interfaces = newDpnInterface.getInterfaces();
1232             }
1233             for (String ifName : interfaces) {
1234                 // In case if there is a InterfacePort in the cache which is not
1235                 // in
1236                 // operational state, skip processing it
1237                 InterfaceInfo ifInfo = interfaceManager
1238                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1239                 if (!isOperational(ifInfo)) {
1240                     continue;
1241                 }
1242
1243                 if (!interfaceManager.isExternalInterface(ifName)) {
1244                     // only add root interfaces
1245                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1246                 }
1247             }
1248
1249             if (listBucket.isEmpty()) { // No Buckets
1250                 createDropBucket(listBucket);
1251             }
1252
1253             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1254             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1255             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1256                     MDSALUtil.buildBucketLists(listBucket));
1257             LOG.trace("installing the localBroadCast Group:{}", group);
1258             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1259         }
1260     }
1261
1262     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1263             InterfaceInfo ifInfo) {
1264         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1265         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1266             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1267                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1268             bucketId++;
1269         }
1270         return bucketId;
1271     }
1272
1273     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1274             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1275             throws ExecutionException, InterruptedException {
1276         Uint64 dpnId = interfaceInfo.getDpId();
1277         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1278         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1279         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1280     }
1281
1282     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1283             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1284             throws ExecutionException, InterruptedException {
1285         Uint64 dpnId = interfaceInfo.getDpId();
1286         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1287         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1288         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1289     }
1290
1291     /**
1292      * Installs a flow in the External Tunnel table consisting in translating
1293      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1294      * elanTag that will be used later in the ELANs pipeline.
1295      * @param dpnId the dpn id
1296      */
1297     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1298             TypedWriteTransaction<Configuration> confTx) {
1299         Uint32 elanTag = elanInfo.getElanTag();
1300         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1301                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1302                 elanInfo.getElanInstanceName(), // flowName
1303                 0, // idleTimeout
1304                 0, // hardTimeout
1305                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1306                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1307                 getInstructionsIntOrExtTunnelTable(elanTag));
1308
1309         mdsalManager.addFlow(confTx, flowEntity);
1310     }
1311
1312     /**
1313      * Removes, from External Tunnel table, the flow that translates from VNI to
1314      * elanTag. Important: ensure this method is only called whenever there is
1315      * no other ElanInterface in the specified DPN
1316      *
1317      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1318      */
1319     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1320             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1321         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1322         // last ElanInstance plus
1323         // adding a new one does (almost at the same time) are executed in that
1324         // exact order
1325
1326         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1327         FlowEntity flowEntity = new FlowEntityBuilder()
1328             .setDpnId(dpnId)
1329             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1330             .setFlowId(flowId)
1331             .build();
1332         mdsalManager.removeFlow(confTx, flowEntity);
1333     }
1334
1335     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1336             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1337         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1338         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1339     }
1340
1341     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1342             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1343         List<? extends MatchInfoBase> listMatchInfoBase;
1344         List<InstructionInfo> instructionInfos;
1345         Uint32 serviceId;
1346         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1347             serviceId = elanTag;
1348             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1349             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1350         } else {
1351             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1352             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1353             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1354         }
1355         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1356                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1357                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1358                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1359                 listMatchInfoBase, instructionInfos);
1360         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1361         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1362                 elanInfo.getElanInstanceName(), elanTag);
1363     }
1364
1365     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1366             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1367         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1368         if (etreeInstance != null) {
1369             setupTerminateServiceTable(elanInfo, dpId,
1370                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1371         }
1372     }
1373
1374     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1375             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1376         long elanTag = elanInfo.getElanTag().toJava();
1377         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1378         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1379         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1380     }
1381
1382     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1383             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1384         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1385         if (etreeLeafTag != null) {
1386             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1387             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1388             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1389         }
1390     }
1391
1392     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1393             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1394         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1395                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1396                 5, elanInfo.getElanInstanceName(), 0, 0,
1397                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1398                 getMatchesForElanTag(elanTag, /* SH flag */false),
1399                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1400
1401         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1402         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1403                 dpId, elanInfo.getElanInstanceName(), elanTag);
1404     }
1405
1406     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1407             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1408         // only if ELAN can connect to external network, perform the following
1409
1410         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1411             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1412                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1413                     5, elanInfo.getElanInstanceName(), 0, 0,
1414                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1415                     getMatchesForElanTag(elanTag, /* SH flag */true),
1416                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1417             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1418             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1419                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1420         }
1421     }
1422
1423
1424     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1425             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1426             throws ExecutionException, InterruptedException {
1427         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1428                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1429         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1430
1431         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1432             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1433                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1434                     .build();
1435             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1436         }
1437     }
1438
1439     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1440         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1441     }
1442
1443     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1444             TypedWriteTransaction<Configuration> tx) {
1445         if (isStandardElanService(elanInterface)) {
1446             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1447                     elanInterface.getName(), lportTag, tx);
1448         } else { // Etree service
1449             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1450         }
1451     }
1452
1453     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1454             TypedWriteTransaction<Configuration> tx) {
1455         int instructionKey = 0;
1456         List<Instruction> instructions = new ArrayList<>();
1457         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1458                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1459
1460         List<Action> actions = new ArrayList<>();
1461         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1462                 lportTag).buildAction());
1463         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1464                 elanTag).buildAction());
1465         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1466
1467         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1468                 ++instructionKey));
1469
1470         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1471         BoundServices serviceInfo = ElanUtils.getBoundServices(
1472                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1473                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1474         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1475         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1476                 bindServiceId);
1477         if (!existingElanService.isPresent()) {
1478             tx.mergeParentStructurePut(bindServiceId, serviceInfo);
1479             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1480         }
1481     }
1482
1483     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1484             TypedWriteTransaction<Configuration> tx) {
1485         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1486             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1487                     lportTag, tx);
1488         } else {
1489             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1490             if (etreeInstance == null) {
1491                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1492                         elanInterface.getName(), elanInfo.getElanInstanceName());
1493             } else {
1494                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1495                         elanInterface.getName(), lportTag, tx);
1496             }
1497         }
1498     }
1499
1500     private static boolean isStandardElanService(ElanInterface elanInterface) {
1501         return elanInterface.augmentation(EtreeInterface.class) == null;
1502     }
1503
1504     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1505             throws ExecutionException, InterruptedException {
1506         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1507         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1508         if (tx.read(bindServiceId).get().isPresent()) {
1509             tx.delete(bindServiceId);
1510         }
1511     }
1512
1513     private static String getFlowRef(long tableId, long elanTag) {
1514         return String.valueOf(tableId) + elanTag;
1515     }
1516
1517     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1518         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1519                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1520     }
1521
1522     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1523         return String.valueOf(tableId) + elanTag + shFlag;
1524     }
1525
1526     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1527         List<Action> listAction = new ArrayList<>();
1528         int actionKey = 0;
1529         listAction.add(
1530             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1531         actionKey++;
1532         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1533         return listAction;
1534     }
1535
1536     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1537             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1538         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1539                 .withKey(new DpnInterfacesKey(dpId)).build();
1540         tx.mergeParentStructurePut(ElanUtils
1541                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface);
1542         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1543                 interfaceNames);
1544         return dpnInterface;
1545     }
1546
1547     /**
1548      * Delete elan dpn interface from operational DS.
1549      *
1550      * @param elanInstanceName
1551      *            the elan instance name
1552      * @param dpId
1553      *            the dp id
1554      */
1555     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1556             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1557         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1558                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1559         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1560         if (dpnInterfaces.isPresent()) {
1561             tx.delete(dpnInterfacesId);
1562         }
1563     }
1564
1565     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1566             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1567         List<String> interfaceNames = new ArrayList<>();
1568         interfaceNames.add(interfaceName);
1569         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1570                 .withKey(new DpnInterfacesKey(dpId)).build();
1571         return dpnInterface;
1572     }
1573
1574     private static void createElanStateList(String elanInstanceName, String interfaceName,
1575             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1576         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1577         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1578         // Adding new Elan Interface Port to the operational DataStore without
1579         // Static-Mac Entries..
1580         if (elanInterfaceLists.isPresent()) {
1581             List<String> interfaceLists = new ArrayList<>();
1582             interfaceLists.add(interfaceName);
1583             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1584             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1585                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1586             }
1587
1588             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1589                     .withKey(new ElanKey(elanInstanceName)).build();
1590             tx.mergeParentStructurePut(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState);
1591             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1592                     interfaceLists);
1593         }
1594     }
1595
1596     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1597         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1598     }
1599
1600     @SuppressWarnings("checkstyle:IllegalCatch")
1601     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1602         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1603         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1604                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1605         if (dpnInterfaceLists == null) {
1606             return;
1607         }
1608         Map<ElanDpnInterfacesListKey, ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists
1609                 .nonnullElanDpnInterfacesList();
1610         for (ElanDpnInterfacesList elanDpns : elanDpnIf.values()) {
1611             int cnt = 0;
1612             String elanName = elanDpns.getElanInstanceName();
1613             ElanInstance elanInfo = elanInstanceCache.get(elanName).orElse(null);
1614             if (elanInfo == null) {
1615                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1616                         + "skipping this ELAN for tunnel handling", elanName);
1617                 continue;
1618             }
1619             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1620                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1621                 continue;
1622             }
1623             Map<DpnInterfacesKey, DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1624             if (dpnInterfaces == null) {
1625                 continue;
1626             }
1627             DpnInterfaces dstDpnIf = null;
1628             for (DpnInterfaces dpnIf : dpnInterfaces.values()) {
1629                 Uint64 dpnIfDpId = dpnIf.getDpId();
1630                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1631                     cnt++;
1632                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1633                     cnt++;
1634                     dstDpnIf = dpnIf;
1635                 }
1636             }
1637             if (cnt == 2) {
1638                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1639                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1640                 jobCoordinator.enqueueJob(elanName, () -> {
1641                     // update Remote BC Group
1642                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1643                     try {
1644                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1645                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1646                                 confTx)).get();
1647                     } catch (RuntimeException e) {
1648                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1649                     }
1650                     Set<String> interfaceLists = new HashSet<>();
1651                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1652                     for (String ifName : interfaceLists) {
1653                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1654                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1655                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1656                             if (isOperational(interfaceInfo)) {
1657                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1658                             }
1659                             return emptyList();
1660                         }, ElanConstants.JOB_MAX_RETRIES);
1661                     }
1662                     return emptyList();
1663                 }, ElanConstants.JOB_MAX_RETRIES);
1664             }
1665
1666         }
1667     }
1668
1669     /**
1670      * Handle external tunnel state event.
1671      *
1672      * @param externalTunnel
1673      *            the external tunnel
1674      * @param intrf
1675      *            the interface
1676      */
1677     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1678         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1679             return;
1680         }
1681         // dpId/externalNodeId will be available either in source or destination
1682         // based on the tunnel end point
1683         Uint64 dpId = null;
1684         NodeId externalNodeId = null;
1685         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1686             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1687             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1688         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1689             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1690             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1691         }
1692         if (dpId == null || externalNodeId == null) {
1693             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1694             return;
1695         }
1696
1697         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1698         if (dpnInterfaceLists == null) {
1699             return;
1700         }
1701         Map<ElanDpnInterfacesListKey, ElanDpnInterfacesList> elanDpnIf
1702                 = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1703         for (ElanDpnInterfacesList elanDpns : elanDpnIf.values()) {
1704             String elanName = elanDpns.getElanInstanceName();
1705             ElanInstance elanInfo = elanInstanceCache.get(elanName).orElse(null);
1706
1707             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1708             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1709                     || dpnInterfaces.getInterfaces().isEmpty()) {
1710                 continue;
1711             }
1712             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1713
1714             final Uint64 finalDpId = dpId;
1715             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1716                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1717                 "Error setting up ELAN BGs");
1718             // install L2gwDevices local macs in dpn.
1719             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1720             // Install dpn macs on external device
1721             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1722                     externalNodeId);
1723         }
1724         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1725     }
1726
1727     /**
1728      * Installs dpn macs in external device. first it checks if the physical
1729      * locator towards this dpn tep is present or not if the physical locator is
1730      * present go ahead and add the ucast macs otherwise update the mcast mac
1731      * entry to include this dpn tep ip and schedule the job to put ucast macs
1732      * once the physical locator is programmed in device
1733      *
1734      * @param elanName
1735      *            the elan name
1736      * @param lstElanInterfaceNames
1737      *            the lst Elan interface names
1738      * @param dpnId
1739      *            the dpn id
1740      * @param externalNodeId
1741      *            the external node id
1742      */
1743     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1744             NodeId externalNodeId) {
1745         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1746                 externalNodeId.getValue());
1747         if (elanL2GwDevice == null) {
1748             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1749             return;
1750         }
1751         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1752         if (dpnTepIp == null) {
1753             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1754                     externalNodeId);
1755             return;
1756         }
1757
1758         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1759         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1760                 LogicalDatastoreType.OPERATIONAL);
1761         boolean phyLocAlreadyExists =
1762                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1763                 dpnTepIp);
1764         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1765                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1766         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1767
1768         if (phyLocAlreadyExists) {
1769             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1770             return;
1771         }
1772         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1773         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1774     }
1775
1776     /**
1777      * Validate external tunnel state event.
1778      *
1779      * @param externalTunnel
1780      *            the external tunnel
1781      * @param intrf
1782      *            the intrf
1783      * @return true, if successful
1784      */
1785     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1786         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1787             String srcDevice = externalTunnel.getDestinationDevice();
1788             String destDevice = externalTunnel.getSourceDevice();
1789             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1790                     LogicalDatastoreType.CONFIGURATION);
1791             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1792                     otherEndPointExtTunnel);
1793             if (otherEndPointExtTunnel != null) {
1794                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1795                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1796                 if (otherEndPointInterfaceOperational) {
1797                     return true;
1798                 } else {
1799                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1800                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1801                 }
1802             }
1803         }
1804         return false;
1805     }
1806
1807     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1808         List<MatchInfo> mkMatches = new ArrayList<>();
1809         // Matching metadata
1810         mkMatches.add(
1811                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1812         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1813         return mkMatches;
1814     }
1815
1816     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1817                                              Uint64 dpId) {
1818         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1819         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1820             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1821                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1822         try {
1823             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1824         } catch (InterruptedException e) {
1825             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1826         }
1827     }
1828 }