elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  14 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  15
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Lists;
  18 import com.google.common.util.concurrent.FluentFuture;
  19 import com.google.common.util.concurrent.ListenableFuture;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Optional;
  29 import java.util.Queue;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.locks.ReentrantLock;
  35 import javax.annotation.PreDestroy;
  36 import javax.inject.Inject;
  37 import javax.inject.Singleton;
  38 import org.apache.commons.lang3.StringUtils;
  39 import org.opendaylight.genius.infra.Datastore.Configuration;
  40 import org.opendaylight.genius.infra.Datastore.Operational;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  44 import org.opendaylight.genius.infra.TypedWriteTransaction;
  45 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  46 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  47 import org.opendaylight.genius.itm.globals.ITMConstants;
  48 import org.opendaylight.genius.mdsalutil.FlowEntity;
  49 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  50 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  51 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  52 import org.opendaylight.genius.mdsalutil.MatchInfo;
  53 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  54 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  55 import org.opendaylight.genius.mdsalutil.NwConstants;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  61 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  62 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  63 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  64 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  65 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  66 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  67 import org.opendaylight.genius.utils.JvmGlobalLocks;
  68 import org.opendaylight.genius.utils.ServiceIndex;
  69 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  70 import org.opendaylight.infrautils.utils.concurrent.Executors;
  71 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  72 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  73 import org.opendaylight.mdsal.binding.api.DataBroker;
  74 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  75 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  76 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  77 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  79 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  80 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  81 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  82 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  83 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  84 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  85 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  86 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  87 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  88 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  89 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  90 import org.opendaylight.serviceutils.srm.RecoverableListener;
  91 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  92 import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesListKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntriesKey;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 142 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 143 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 144 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 145 import org.opendaylight.yangtools.yang.common.Uint32;
 146 import org.opendaylight.yangtools.yang.common.Uint64;
 147 import org.slf4j.Logger;
 148 import org.slf4j.LoggerFactory;
 149
 150 /**
 151  * Class in charge of handling creations, modifications and removals of
 152  * ElanInterfaces.
 153  *
 154  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 155  */
 156 @Singleton
 157 public class ElanInterfaceManager extends AbstractAsyncDataTreeChangeListener<ElanInterface>
 158         implements RecoverableListener {
 159     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 160     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 161     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 162     private static final boolean SH_FLAG_SET = true;
 163     private static final boolean SH_FLAG_UNSET = false;
 164
 165     private final DataBroker broker;
 166     private final ManagedNewTransactionRunner txRunner;
 167     private final IMdsalApiManager mdsalManager;
 168     private final IInterfaceManager interfaceManager;
 169     private final IdManagerService idManager;
 170     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 171     private final INeutronVpnManager neutronVpnManager;
 172     private final ElanItmUtils elanItmUtils;
 173     private final ElanEtreeUtils elanEtreeUtils;
 174     private final ElanL2GatewayUtils elanL2GatewayUtils;
 175     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 176     private final ElanUtils elanUtils;
 177     private final JobCoordinator jobCoordinator;
 178     private final ElanInstanceCache elanInstanceCache;
 179     private final ElanInterfaceCache elanInterfaceCache;
 180     private final ElanGroupCache elanGroupCache;
 181
 182     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 183         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 184
 185     @Inject
 186     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 187                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 188                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 189                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 190                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 191                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 192                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 193                                 final ElanInstanceCache elanInstanceCache,
 194                                 final ElanInterfaceCache elanInterfaceCache,
 195                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 196                                 ElanGroupCache elanGroupCache,
 197                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 198         super(dataBroker, LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ElanInterfaces.class)
 199                 .child(ElanInterface.class),
 200                 Executors.newListeningSingleThreadExecutor("ElanInterfaceManager", LOG));
 201         this.broker = dataBroker;
 202         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 203         this.idManager = managerService;
 204         this.mdsalManager = mdsalApiManager;
 205         this.interfaceManager = interfaceManager;
 206         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 207         this.neutronVpnManager = neutronVpnManager;
 208         this.elanItmUtils = elanItmUtils;
 209         this.elanEtreeUtils = elanEtreeUtils;
 210         this.elanL2GatewayUtils = elanL2GatewayUtils;
 211         this.elanUtils = elanUtils;
 212         this.jobCoordinator = jobCoordinator;
 213         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 214         this.elanInstanceCache = elanInstanceCache;
 215         this.elanInterfaceCache = elanInterfaceCache;
 216         this.elanGroupCache = elanGroupCache;
 217         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 218     }
 219
 220     public void init() {
 221         LOG.info("{} registered", getClass().getSimpleName());
 222     }
 223
 224     @Override
 225     public void registerListener() {
 226         super.register();
 227         LOG.info("Registering ElanInterfaceManager");
 228     }
 229
 230     @Override
 231     public void deregisterListener() {
 232         super.close();
 233         LOG.info("Deregistering ElanInterfaceManager");
 234     }
 235
 236     @Override
 237     @PreDestroy
 238     public void close() {
 239         super.close();
 240         Executors.shutdownAndAwaitTermination(getExecutorService());
 241     }
 242
 243     @Override
 244     public void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 245         String interfaceName = del.getName();
 246         String elanInstanceName = del.getElanInstanceName();
 247         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 248         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 249         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 250             elanInterfaces.remove(del);
 251             if (elanInterfaces.isEmpty()) {
 252                 unProcessedElanInterfaces.remove(elanInstanceName);
 253             }
 254         }
 255         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orElse(null);
 256         /*
 257          * Handling in case the elan instance is deleted.If the Elan instance is
 258          * deleted, there is no need to explicitly delete the elan interfaces
 259          */
 260         if (elanInfo == null) {
 261             return;
 262         }
 263         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 264         if (interfaceInfo == null && elanInfo.isExternal()) {
 265             // In deleting external network, the underlying ietf Inteface might have been removed
 266             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 267             // from Operational DS instead
 268             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 269         }
 270         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 271                 interfaceName, interfaceInfo, this);
 272         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 273     }
 274
 275     private static class RemoveElanInterfaceHolder {
 276         boolean isLastElanInterface = false;
 277         Uint64 dpId = null;
 278     }
 279
 280     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 281     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 282             InterfaceInfo interfaceInfo) {
 283         String elanName = elanInfo.getElanInstanceName();
 284         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 285         Uint32 elanTag = elanInfo.getElanTag();
 286         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 287         List<ListenableFuture<Void>> futures = new ArrayList<>();
 288         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 289         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 290             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 291             if (elanState == null) {
 292                 return;
 293             }
 294             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 295                 List<String> elanInterfaces = elanState.getElanInterfaces();
 296                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 297                     holder.isLastElanInterface = true;
 298                 }
 299                 if (interfaceInfo != null) {
 300                     holder.dpId = interfaceInfo.getDpId();
 301                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 302                         interfaceName, elanTag, interfaceTx);
 303                     /*
 304                      * If there are not elan ports, remove the unknown dmac, terminating
 305                      * service table flows, remote/local bc group
 306                      */
 307                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 308                         || dpnInterfaces.getInterfaces().isEmpty()) {
 309                         // No more Elan Interfaces in this DPN
 310                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 311                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 312                             holder.dpId);
 313                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 314                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 315                         }
 316                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 317                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 318                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 319                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 320                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 321                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 322                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 323                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 324                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 325                             }
 326                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 327                         }
 328                     } else {
 329                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 330                     }
 331                 }
 332             }));
 333         }));
 334         futures.forEach(ElanUtils::waitForTransactionToComplete);
 335
 336         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 337                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 338         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 339                 ElanConstants.JOB_MAX_RETRIES);
 340
 341         return futures;
 342     }
 343
 344     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 345             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 346             throws ExecutionException, InterruptedException {
 347         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 348         if (etreeLeafTag != null) {
 349             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 350             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 351         }
 352     }
 353
 354     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 355             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 356             throws ExecutionException, InterruptedException {
 357         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 358         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 359     }
 360
 361     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 362             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 363             throws ExecutionException, InterruptedException {
 364         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 365         if (etreeInstance != null) {
 366             Uint64 dpnId = interfaceInfo.getDpId();
 367             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 368             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 369             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 370         }
 371     }
 372
 373     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 374             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 375             throws ExecutionException, InterruptedException {
 376         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 377         if (etreeInstance != null) {
 378             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 379             Uint64 dpnId = interfaceInfo.getDpId();
 380             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 381             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 382             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 383         }
 384     }
 385
 386     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 387             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 388         String elanName = elanInfo.getElanInstanceName();
 389         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 390         if (elanState == null) {
 391             return elanState;
 392         }
 393         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 394         List<String> elanInterfaces = new ArrayList<>();
 395         if (existingElanInterfaces != null) {
 396             elanInterfaces.addAll(existingElanInterfaces);
 397         }
 398         boolean isRemoved = elanInterfaces.remove(interfaceName);
 399         if (!isRemoved) {
 400             return elanState;
 401         }
 402
 403         if (elanInterfaces.isEmpty()) {
 404             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 405             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 406             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 407             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 408         } else {
 409             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 410                     .withKey(new ElanKey(elanName)).build();
 411             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 412         }
 413         return elanState;
 414     }
 415
 416     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 417             throws ExecutionException, InterruptedException {
 418         // removing the ElanInterface from the config data_store if interface is
 419         // not present in Interface config DS
 420         InstanceIdentifier<ElanInterface> elanInterfaceId = ElanUtils
 421                 .getElanInterfaceConfigurationDataPathId(interfaceName);
 422         FluentFuture<Optional<ElanInterface>> interfaceOptional = tx.read(elanInterfaceId);
 423         if (!interfaceOptional.get().isPresent() && elanInterfaceCache.get(interfaceName).isPresent()) {
 424             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 425         }
 426     }
 427
 428     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 429             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 430         String elanName = elanInfo.getElanInstanceName();
 431         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 432         List<ListenableFuture<Void>> futures = new ArrayList<>();
 433         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 434             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 435                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 436                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 437                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 438                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 439                 if (interfaceInfo != null) {
 440                     if (existingElanInterfaceMac.isPresent()) {
 441                         Map<MacEntryKey, MacEntry> existingMacEntries =
 442                             existingElanInterfaceMac.get().nonnullMacEntry();
 443                         if (existingMacEntries != null) {
 444                             List<PhysAddress> macAddresses = new ArrayList<>();
 445                             for (MacEntry macEntry : existingMacEntries.values()) {
 446                                 PhysAddress macAddress = macEntry.getMacAddress();
 447                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 448                                         macAddress.getValue(), interfaceName);
 449                                 Optional<MacEntry> macEntryOptional =
 450                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 451                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 452                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 453                                 }
 454                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 455                                         interfaceName, elanName, macAddress);
 456                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 457                                 macAddresses.add(macAddress);
 458                             }
 459
 460                             // Removing all those MACs from External Devices belonging
 461                             // to this ELAN
 462                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 463                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 464                             }
 465                         }
 466                     }
 467                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 468                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 469                 } else if (existingElanInterfaceMac.isPresent()) {
 470                     // Interface does not exist in ConfigDS, so lets remove everything
 471                     // about that interface related to Elan
 472                     Map<MacEntryKey, MacEntry> macEntries = existingElanInterfaceMac.get().nonnullMacEntry();
 473                     if (macEntries != null) {
 474                         for (MacEntry macEntry : macEntries.values()) {
 475                             PhysAddress macAddress = macEntry.getMacAddress();
 476                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 477                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 478                             }
 479                         }
 480                     }
 481                 }
 482                 if (existingElanInterfaceMac.isPresent()) {
 483                     interfaceTx.delete(elanInterfaceId);
 484                 }
 485                 unbindService(interfaceName, flowTx);
 486                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 487             }));
 488         }));
 489         return futures;
 490     }
 491
 492     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 493                                                                          String interfaceName, Uint32 elanTag,
 494                                                                          TypedReadWriteTransaction<Operational> tx)
 495             throws ExecutionException, InterruptedException {
 496         // FIXME: pass in and use ElanInstanceKey instead?
 497         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 498         lock.lock();
 499         try {
 500             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 501             if (dpnInterfaces != null) {
 502                 List<String> interfaceLists = null;
 503                 if (dpnInterfaces.getInterfaces() != null) {
 504                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 505                 }
 506                 if (interfaceLists != null) {
 507                     interfaceLists.remove(interfaceName);
 508                 }
 509
 510                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 511                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 512                     deleteElanDpnInterface(elanName, dpId, tx);
 513                 } else {
 514                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 515                 }
 516             }
 517             return dpnInterfaces;
 518         } finally {
 519             lock.unlock();
 520         }
 521     }
 522
 523     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 524         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 525         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 526             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 527                 Uint64 currentDpId = dpnInterface.getDpId();
 528                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 529                     for (String elanInterface : dpnInterface.getInterfaces()) {
 530                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 531                         if (macs == null || macs.getMacEntry() == null) {
 532                             continue;
 533                         }
 534                         for (MacEntry mac : new ArrayList<MacEntry>(macs.nonnullMacEntry().values())) {
 535                             removeTheMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 536                             removeEtreeMacFlowInTheDPN(dpId, elanTag, mac, confTx);
 537                         }
 538                     }
 539                 }
 540             }
 541         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 542     }
 543
 544     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 545             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 546         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 547         if (etreeLeafTag != null) {
 548             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(), mac, confTx);
 549         }
 550     }
 551
 552     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, MacEntry mac,
 553             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 554         mdsalManager
 555                 .removeFlow(confTx, dpId,
 556                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 557                                 ElanUtils.getKnownDynamicmacFlowRef(elanTag, mac.getMacAddress().getValue())));
 558     }
 559
 560     /*
 561     * Possible Scenarios for update
 562     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 563         then all {1,2,3,4} should be removed
 564
 565         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 566         then all {1,2,3,4} should be added
 567
 568         c. if orig = {1,2,3,4} updated={2,3,4}
 569         then 1 should be removed
 570
 571         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 572         then we should just add 5
 573
 574         e. if orig = {1,2,3,4} updated={2,3,4,5}
 575         then 1 should be removed , 5 should be added
 576     * */
 577     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 578     @Override
 579     public void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 580         // updating the static-Mac Entries for the existing elanInterface
 581         String elanName = update.getElanInstanceName();
 582         String interfaceName = update.getName();
 583         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 584         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 585
 586         List<StaticMacEntries> originalStaticMacEntries = new ArrayList<StaticMacEntries>(original
 587                 .nonnullStaticMacEntries().values());
 588         List<StaticMacEntries> updatedStaticMacEntries = new ArrayList<StaticMacEntries>(update
 589                 .nonnullStaticMacEntries().values());
 590         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 591         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 592
 593         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 594                 deletedEntry.getMacAddress()));
 595
 596         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 597         * Hence add the macentries for the same.*/
 598         for (StaticMacEntries staticMacEntry : updatedEntries) {
 599             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 600                     staticMacEntry.getMacAddress());
 601             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 602                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 603                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 604                     if (existingMacEntry.isPresent()) {
 605                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 606                                 existingMacEntry.get(), elanName);
 607                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 608                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 609                             tx);
 610                     } else {
 611                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 612                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 613                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 614                             elanName, interfaceName, staticMacEntry, tx);
 615                     }
 616                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 617         }
 618     }
 619
 620     @Override
 621     public void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 622         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 623         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 624             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 625             String interfaceName = elanInterfaceAdded.getName();
 626             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 627             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 628             if (interfaceInfo == null) {
 629                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 630                 return;
 631             }
 632             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orElse(null);
 633
 634             if (elanInstance == null) {
 635                 // Add the ElanInstance in the Configuration data-store
 636                 List<String> elanInterfaces = new ArrayList<>();
 637                 elanInterfaces.add(interfaceName);
 638                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 639                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 640                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 641                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 642             }
 643
 644             Uint32 elanTag = elanInstance.getElanTag();
 645             // If elan tag is not updated, then put the elan interface into
 646             // unprocessed entry map and entry. Let entries
 647             // in this map get processed during ELAN update DCN.
 648             if (elanTag == null || elanTag.longValue() == 0L) {
 649                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 650                 if (elanInterfaces == null) {
 651                     elanInterfaces = new ConcurrentLinkedQueue<>();
 652                 }
 653                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 654                     elanInterfaces.add(elanInterfaceAdded);
 655                 }
 656                 LOG.error("ELAN tag for instance {} is not created. Adding it to unprocessed list."
 657                         + " Recreate the network if this message is seen multiple times", elanInstanceName);
 658                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 659                 return;
 660             }
 661             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 662                     interfaceInfo, elanInstance, this);
 663             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 664         }), LOG, "Error processing added ELAN interface");
 665     }
 666
 667     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 668         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 669         List<ListenableFuture<Void>> futures = new ArrayList<>();
 670         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 671         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 672             return futures;
 673         }
 674         for (ElanInterface elanInterface : elanInterfaces) {
 675             String interfaceName = elanInterface.getName();
 676             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 677             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 678         }
 679         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 680         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 681                 elanInstance.getElanInstanceName());
 682         return futures;
 683     }
 684
 685     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 686             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 687         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 688                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 689         List<DpnInterfaces> dpnInterfaceLists = null;
 690         if (elanDpnInterfacesList != null) {
 691             dpnInterfaceLists = new ArrayList<DpnInterfaces>(elanDpnInterfacesList.nonnullDpnInterfaces().values());
 692         }
 693         if (dpnInterfaceLists != null && !dpnInterfaceLists.isEmpty()) {
 694             Uint64 dstDpId = interfaceInfo.getDpId();
 695             processRemoteDmacFlowForInterface(dstDpId, elanInstance, dpnInterfaceLists, writeFlowGroupTx);
 696         }
 697     }
 698
 699     private void processRemoteDmacFlowForInterface(Uint64 dstDpId, ElanInstance elanInstance,
 700              List<DpnInterfaces> dpnInterfaceLists, TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 701         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 702             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 703                 continue;
 704             }
 705             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 706             if (remoteElanInterfaces == null || remoteElanInterfaces.isEmpty()) {
 707                 continue;
 708             }
 709             for (String remoteIf : remoteElanInterfaces) {
 710                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 711                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 712                 if (elanIfMac == null || remoteInterface == null) {
 713                     continue;
 714                 }
 715                 Map<MacEntryKey, MacEntry> remoteMacEntries = elanIfMac.nonnullMacEntry();
 716                 for (MacEntry macEntry : remoteMacEntries.values()) {
 717                     String macAddress = macEntry.getMacAddress().getValue();
 718                     LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 719                             dstDpId, elanInstance.getElanInstanceName());
 720                     elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 721                             remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 722                             elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 723                 }
 724             }
 725         }
 726     }
 727
 728     private static class AddElanInterfaceHolder {
 729         private DpnInterfaces dpnInterfaces = null;
 730         private boolean isFirstInterfaceInDpn = false;
 731         private Uint64 dpId;
 732     }
 733
 734     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 735     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 736             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 737         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 738         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 739         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 740
 741         String interfaceName = elanInterface.getName();
 742         String elanInstanceName = elanInterface.getElanInstanceName();
 743         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 744         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 745
 746         List<ListenableFuture<Void>> futures = new ArrayList<>();
 747         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 748         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 749             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 750             if (elanInfo == null) {
 751                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 752                 List<String> elanInterfaces = new ArrayList<>();
 753                 elanInterfaces.add(interfaceName);
 754                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 755                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 756                         operTx)));
 757             } else {
 758                 createElanStateList(elanInstanceName, interfaceName, operTx);
 759             }
 760             // Specific actions to the DPN where the ElanInterface has been added,
 761             // for example, programming the
 762             // External tunnel table if needed or adding the ElanInterface to the
 763             // DpnInterfaces in the operational DS.
 764             holder.dpId = interfaceInfo.getDpId();
 765             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 766                 // FIXME: use elanInstaince.key() instead?
 767                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 768                 lock.lock();
 769                 try {
 770                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 771                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 772                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 773                     if (ElanUtils.isVlan(elanInstance)) {
 774                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 775                             elanInstanceName, existingElanDpnInterfaces);
 776                     } else {
 777                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 778                     }
 779                     if (holder.isFirstInterfaceInDpn) {
 780                         // ELAN's 1st ElanInterface added to this DPN
 781                         if (!existingElanDpnInterfaces.isPresent()) {
 782                             holder.dpnInterfaces =
 783                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 784                         } else {
 785                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 786                             List<String> elanInterfaces =
 787                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 788                             elanInterfaces.add(interfaceName);
 789                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 790                                 elanInterfaces, operTx);
 791                         }
 792                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 793                                 interfaceName, elanInstanceName, holder.dpId);
 794                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 795                         // table, but only if Elan has VNI
 796                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 797                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 798                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 799                         }
 800                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 801                             interfaceName);
 802                     } else {
 803                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 804                         List<String> elanInterfaces =
 805                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 806                         elanInterfaces.add(interfaceName);
 807                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 808                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 809                                 interfaceName);
 810                         }
 811                         holder.dpnInterfaces =
 812                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 813                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 814                                 interfaceName, elanInstanceName, holder.dpId);
 815                     }
 816                 } finally {
 817                     lock.unlock();
 818                 }
 819             }
 820
 821             // add code to install Local/Remote BC group, unknow DMAC entry,
 822             // terminating service table flow entry
 823             // call bindservice of interfacemanager to create ingress table flow
 824             // enty.
 825         }));
 826         futures.forEach(ElanUtils::waitForTransactionToComplete);
 827         futures.add(
 828             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 829                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 830                     holder.isFirstInterfaceInDpn, confTx))));
 831
 832         // add the vlan provider interface to remote BC group for the elan
 833         // for internal vlan networks
 834         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 835             if (interfaceManager.isExternalInterface(interfaceName)) {
 836                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 837                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 838             }
 839         }
 840         if (holder.isFirstInterfaceInDpn) {
 841             // ELAN's 1st ElanInterface added to this DPN
 842             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 843             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 844                 operTx.mergeParentStructurePut(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName,
 845                         holder.dpId), holder.dpnInterfaces);
 846             }));
 847         } else {
 848             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 849         }
 850
 851         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 852                 holder.isFirstInterfaceInDpn, elanInterface);
 853         return futures;
 854     }
 855
 856     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 857     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 858             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 859         String elanInstanceName = elanInstance.getElanInstanceName();
 860         String interfaceName = elanInterface.getName();
 861         List<ListenableFuture<Void>> futures = new ArrayList<>();
 862         Uint64 dpId = interfaceInfo.getDpId();
 863         boolean isInterfaceOperational = isOperational(interfaceInfo);
 864         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 865             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 866                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 867                     isFirstInterfaceInDpn, confTx);
 868
 869                 Map<StaticMacEntriesKey, StaticMacEntries> staticMacEntriesList =
 870                     elanInterface.nonnullStaticMacEntries();
 871                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 872
 873                 if (ElanUtils.isNotEmpty(staticMacEntriesList.values())) {
 874                     for (StaticMacEntries staticMacEntry : staticMacEntriesList.values()) {
 875                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 876                             staticMacEntry.getMacAddress());
 877                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 878                             LogicalDatastoreType.OPERATIONAL, macId);
 879                         if (existingMacEntry.isPresent()) {
 880                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 881                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 882                                 existingMacEntry.get(), operTx);
 883                         } else {
 884                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 885                                 interfaceName, staticMacEntry, operTx);
 886                         }
 887
 888                         if (isInterfaceOperational) {
 889                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 890                             // DPNs
 891                             String macAddress = staticMacEntry.getMacAddress().getValue();
 892                             LOG.info(
 893                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 894                                     + " {}",
 895                                 macAddress, elanInstanceName, interfaceName);
 896                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 897                                     interfaceName, elanInstanceName, macAddress);
 898                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 899                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 900                         }
 901                     }
 902
 903                     if (isInterfaceOperational) {
 904                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 905                         // on purpose.
 906                         for (StaticMacEntries staticMacEntry : staticMacEntriesList.values()) {
 907                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 908                         }
 909                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 910                             staticMacAddresses);
 911                     }
 912                 }
 913             }));
 914         }));
 915         futures.forEach(ElanUtils::waitForTransactionToComplete);
 916         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 917             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 918             try {
 919                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 920                 if (neutronPort != null) {
 921                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 922                 }
 923             } catch (IllegalArgumentException ex) {
 924                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 925             }
 926         }
 927         return futures;
 928     }
 929
 930     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 931             PhysAddress physAddress) {
 932         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 933         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 934         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 935                 LogicalDatastoreType.OPERATIONAL, macId);
 936
 937         if (!existingMacEntry.isPresent()) {
 938             return;
 939         }
 940
 941         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 942                 .withKey(new MacEntryKey(physAddress)).build();
 943         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 944                 elanInstanceCache.get(elanInstanceName).orElse(null), interfaceInfo, macEntry);
 945     }
 946
 947     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 948             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 949         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 950         if (!existingElanDpnInterfaces.isPresent()) {
 951             return true;
 952         }
 953         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 954             return false;
 955         }
 956         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 957         int dummyInterfaceCount =  0;
 958         List<String> interfaces = dpnInterfaces.getInterfaces();
 959         if (interfaces == null) {
 960             return true;
 961         }
 962         if (interfaces.contains(routerPortUuid)) {
 963             dummyInterfaceCount++;
 964         }
 965         if (interfaces.contains(elanInstanceName)) {
 966             dummyInterfaceCount++;
 967         }
 968         return interfaces.size() == dummyInterfaceCount;
 969     }
 970
 971     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 972             PhysAddress physAddress) {
 973         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 974                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 975     }
 976
 977     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 978             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 979         if (!isOperational(interfaceInfo)) {
 980             LOG.warn("Interface {} is not operational", elanInterface.getName());
 981             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 982                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 983             return;
 984         }
 985         Uint64 dpId = interfaceInfo.getDpId();
 986         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 987             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 988         }
 989         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 990         if (isFirstInterfaceInDpn) {
 991             // Terminating Service , UnknownDMAC Table.
 992             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 993             // for ELAN Instance is VxLAN
 994             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 995                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 996             }
 997             setupUnknownDMacTable(elanInstance, dpId, confTx);
 998             /*
 999              * Install remote DMAC flow. This is required since this DPN is
1000              * added later to the elan instance and remote DMACs of other
1001              * interfaces in this elan instance are not present in the current
1002              * dpn.
1003              */
1004             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
1005                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
1006                         elanInstance.getElanInstanceName());
1007                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
1008             }
1009         }
1010         // bind the Elan service to the Interface
1011         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
1012     }
1013
1014     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1015             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
1016         if (!isOperational(interfaceInfo)) {
1017             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
1018                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1019             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1020                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1021             return;
1022         }
1023         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1024                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1025         // LocalBroadcast Group creation with elan-Interfaces
1026         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1027                 dpnInterfaces.getDpId());
1028         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1029         if (isFirstInterfaceInDpn) {
1030             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1031             try {
1032                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1033             } catch (InterruptedException e1) {
1034                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1035             }
1036         }
1037     }
1038
1039     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1040         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1041         return InstanceIdentifier.builder(Nodes.class)
1042                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1043                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1044                 .augmentation(FlowCapableNode.class)
1045                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1046     }
1047
1048     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1049                                                               InterfaceInfo interfaceInfo,
1050                                                               DpnInterfaces dpnInterfaces,
1051                                                               boolean isFirstInterfaceInDpn,
1052                                                               ElanInterface elanInterface) {
1053         if (!isOperational(interfaceInfo)) {
1054             LOG.debug("Interface {} is not operational", elanInterface.getName());
1055             return;
1056         }
1057         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1058         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1059                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1060         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1061         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1062             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1063         });
1064     }
1065
1066     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1067             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1068         int ifTag = interfaceInfo.getInterfaceTag();
1069         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1070                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 11, elanInfo.getElanInstanceName(),
1071                 0, 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1072                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1073                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1074
1075         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1076         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1077                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1078         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<InstructionKey, Instruction>();
1079         int instructionKey = 0;
1080         for (Instruction instructionObj : MDSALUtil.buildInstructionsDrop()) {
1081             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
1082         }
1083
1084         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1085                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 12, elanInfo.getElanInstanceName(), 0,
1086                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1087                 getMatchesForFilterEqualsLPortTag(ifTag), customInstructionsMap);
1088
1089         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1090         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1091                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1092     }
1093
1094     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1095             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1096         int ifTag = interfaceInfo.getInterfaceTag();
1097         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1098             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1099
1100         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1101
1102         Flow flowEntity = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1103             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"));
1104
1105         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1106     }
1107
1108     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1109         List<MatchInfo> mkMatches = new ArrayList<>();
1110         MatchInfo match = new MatchTunnelId(vni);
1111         mkMatches.add(match);
1112         return mkMatches;
1113     }
1114
1115     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1116         List<InstructionInfo> mkInstructions = new ArrayList<>();
1117         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1118         return mkInstructions;
1119     }
1120
1121     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1122         List<MatchInfo> mkMatches = new ArrayList<>();
1123         // Matching metadata
1124         mkMatches.add(new MatchMetadata(
1125                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1126         return mkMatches;
1127     }
1128
1129     /**
1130      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1131      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1132      *
1133      * @param elanTag
1134      *            elanTag to be written in metadata when flow is selected
1135      * @return the instructions ready to be installed in a flow
1136      */
1137     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1138         List<InstructionInfo> mkInstructions = new ArrayList<>();
1139         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1140                 ElanHelper.getElanMetadataMask()));
1141         /* applicable for EXTERNAL_TUNNEL_TABLE only
1142         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1143         */
1144         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1145         return mkInstructions;
1146     }
1147
1148     // Install DMAC entry on dst DPN
1149     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1150             Uint64 dstDpId) {
1151         String interfaceName = interfaceInfo.getInterfaceName();
1152         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1153         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1154             Map<MacEntryKey, MacEntry> macEntries = elanInterfaceMac.nonnullMacEntry();
1155             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1156                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1157                     for (MacEntry macEntry : macEntries.values()) {
1158                         String macAddress = macEntry.getMacAddress().getValue();
1159                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1160                             interfaceName);
1161                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1162                             interfaceInfo.getDpId())) {
1163                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1164                                 macAddress);
1165                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1166                         }
1167                     }
1168                 })));
1169         }
1170         return emptyList();
1171     }
1172
1173     private static void createDropBucket(List<Bucket> listBucket) {
1174         List<Action> actionsInfos = new ArrayList<>();
1175         actionsInfos.add(new ActionDrop().buildAction());
1176         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1177                 MDSALUtil.WATCH_GROUP);
1178         listBucket.add(dropBucket);
1179     }
1180
1181     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1182             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1183         if (!isOperational(interfaceInfo)) {
1184             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1185                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1186             return;
1187         }
1188         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1189         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1190     }
1191
1192     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1193             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1194         List<Bucket> listBucket = new ArrayList<>();
1195         int bucketId = 0;
1196         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1197
1198         List<String> interfaces = new ArrayList<>();
1199         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1200             interfaces = newDpnInterface.getInterfaces();
1201         }
1202         for (String ifName : interfaces) {
1203             // In case if there is a InterfacePort in the cache which is not in
1204             // operational state, skip processing it
1205             InterfaceInfo ifInfo = interfaceManager
1206                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1207             if (!isOperational(ifInfo)) {
1208                 continue;
1209             }
1210
1211             if (!interfaceManager.isExternalInterface(ifName)) {
1212                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1213                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1214                 bucketId++;
1215             }
1216         }
1217
1218         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1219                 MDSALUtil.buildBucketLists(listBucket));
1220         LOG.trace("installing the localBroadCast Group:{}", group);
1221         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1222     }
1223
1224     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1225             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1226         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1227         if (etreeInstance != null) {
1228             List<Bucket> listBucket = new ArrayList<>();
1229             int bucketId = 0;
1230
1231             List<String> interfaces = new ArrayList<>();
1232             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1233                 interfaces = newDpnInterface.getInterfaces();
1234             }
1235             for (String ifName : interfaces) {
1236                 // In case if there is a InterfacePort in the cache which is not
1237                 // in
1238                 // operational state, skip processing it
1239                 InterfaceInfo ifInfo = interfaceManager
1240                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1241                 if (!isOperational(ifInfo)) {
1242                     continue;
1243                 }
1244
1245                 if (!interfaceManager.isExternalInterface(ifName)) {
1246                     // only add root interfaces
1247                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1248                 }
1249             }
1250
1251             if (listBucket.isEmpty()) { // No Buckets
1252                 createDropBucket(listBucket);
1253             }
1254
1255             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1256             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1257             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1258                     MDSALUtil.buildBucketLists(listBucket));
1259             LOG.trace("installing the localBroadCast Group:{}", group);
1260             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1261         }
1262     }
1263
1264     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1265             InterfaceInfo ifInfo) {
1266         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1267         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1268             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1269                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1270             bucketId++;
1271         }
1272         return bucketId;
1273     }
1274
1275     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1276             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1277             throws ExecutionException, InterruptedException {
1278         Uint64 dpnId = interfaceInfo.getDpId();
1279         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1280         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1281         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1282     }
1283
1284     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1285             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1286             throws ExecutionException, InterruptedException {
1287         Uint64 dpnId = interfaceInfo.getDpId();
1288         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1289         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1290         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1291     }
1292
1293     /**
1294      * Installs a flow in the External Tunnel table consisting in translating
1295      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1296      * elanTag that will be used later in the ELANs pipeline.
1297      * @param dpnId the dpn id
1298      */
1299     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1300             TypedWriteTransaction<Configuration> confTx) {
1301         Uint32 elanTag = elanInfo.getElanTag();
1302         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1303                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1304                 elanInfo.getElanInstanceName(), // flowName
1305                 0, // idleTimeout
1306                 0, // hardTimeout
1307                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1308                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1309                 getInstructionsIntOrExtTunnelTable(elanTag));
1310
1311         mdsalManager.addFlow(confTx, flowEntity);
1312     }
1313
1314     /**
1315      * Removes, from External Tunnel table, the flow that translates from VNI to
1316      * elanTag. Important: ensure this method is only called whenever there is
1317      * no other ElanInterface in the specified DPN
1318      *
1319      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1320      */
1321     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1322             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1323         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1324         // last ElanInstance plus
1325         // adding a new one does (almost at the same time) are executed in that
1326         // exact order
1327
1328         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1329         FlowEntity flowEntity = new FlowEntityBuilder()
1330             .setDpnId(dpnId)
1331             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1332             .setFlowId(flowId)
1333             .build();
1334         mdsalManager.removeFlow(confTx, flowEntity);
1335     }
1336
1337     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1338             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1339         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1340         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1341     }
1342
1343     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1344             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1345         List<? extends MatchInfoBase> listMatchInfoBase;
1346         List<InstructionInfo> instructionInfos;
1347         Uint32 serviceId;
1348         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1349             serviceId = elanTag;
1350             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1351             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1352         } else {
1353             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1354             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1355             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1356         }
1357         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1358                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1359                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1360                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1361                 listMatchInfoBase, instructionInfos);
1362         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1363         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1364                 elanInfo.getElanInstanceName(), elanTag);
1365     }
1366
1367     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1368             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1369         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1370         if (etreeInstance != null) {
1371             setupTerminateServiceTable(elanInfo, dpId,
1372                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1373         }
1374     }
1375
1376     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1377             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1378         long elanTag = elanInfo.getElanTag().toJava();
1379         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1380         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1381         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1382     }
1383
1384     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1385             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1386         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1387         if (etreeLeafTag != null) {
1388             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1389             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1390             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1391         }
1392     }
1393
1394     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1395             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1396         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1397                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1398                 5, elanInfo.getElanInstanceName(), 0, 0,
1399                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1400                 getMatchesForElanTag(elanTag, /* SH flag */false),
1401                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1402
1403         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1404         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1405                 dpId, elanInfo.getElanInstanceName(), elanTag);
1406     }
1407
1408     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1409             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1410         // only if ELAN can connect to external network, perform the following
1411
1412         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1413             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1414                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1415                     5, elanInfo.getElanInstanceName(), 0, 0,
1416                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1417                     getMatchesForElanTag(elanTag, /* SH flag */true),
1418                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1419             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1420             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1421                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1422         }
1423     }
1424
1425
1426     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1427             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1428             throws ExecutionException, InterruptedException {
1429         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1430                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1431         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1432
1433         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1434             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1435                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1436                     .build();
1437             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1438         }
1439     }
1440
1441     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1442         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1443     }
1444
1445     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1446             TypedWriteTransaction<Configuration> tx) {
1447         if (isStandardElanService(elanInterface)) {
1448             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1449                     elanInterface.getName(), lportTag, tx);
1450         } else { // Etree service
1451             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1452         }
1453     }
1454
1455     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1456             TypedWriteTransaction<Configuration> tx) {
1457         int instructionKey = 0;
1458         List<Instruction> instructions = new ArrayList<>();
1459         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1460                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1461
1462         List<Action> actions = new ArrayList<>();
1463         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1464                 lportTag).buildAction());
1465         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1466                 elanTag).buildAction());
1467         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1468
1469         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1470                 ++instructionKey));
1471
1472         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1473         BoundServices serviceInfo = ElanUtils.getBoundServices(
1474                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1475                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1476         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1477         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1478                 bindServiceId);
1479         if (!existingElanService.isPresent()) {
1480             tx.mergeParentStructurePut(bindServiceId, serviceInfo);
1481             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1482         }
1483     }
1484
1485     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1486             TypedWriteTransaction<Configuration> tx) {
1487         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1488             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1489                     lportTag, tx);
1490         } else {
1491             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1492             if (etreeInstance == null) {
1493                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1494                         elanInterface.getName(), elanInfo.getElanInstanceName());
1495             } else {
1496                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1497                         elanInterface.getName(), lportTag, tx);
1498             }
1499         }
1500     }
1501
1502     private static boolean isStandardElanService(ElanInterface elanInterface) {
1503         return elanInterface.augmentation(EtreeInterface.class) == null;
1504     }
1505
1506     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1507             throws ExecutionException, InterruptedException {
1508         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1509         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1510         if (tx.read(bindServiceId).get().isPresent()) {
1511             tx.delete(bindServiceId);
1512         }
1513     }
1514
1515     private static String getFlowRef(long tableId, long elanTag) {
1516         return String.valueOf(tableId) + elanTag;
1517     }
1518
1519     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1520         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1521                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1522     }
1523
1524     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1525         return String.valueOf(tableId) + elanTag + shFlag;
1526     }
1527
1528     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1529         List<Action> listAction = new ArrayList<>();
1530         int actionKey = 0;
1531         listAction.add(
1532             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1533         actionKey++;
1534         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1535         return listAction;
1536     }
1537
1538     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1539             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1540         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1541                 .withKey(new DpnInterfacesKey(dpId)).build();
1542         tx.mergeParentStructurePut(ElanUtils
1543                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface);
1544         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1545                 interfaceNames);
1546         return dpnInterface;
1547     }
1548
1549     /**
1550      * Delete elan dpn interface from operational DS.
1551      *
1552      * @param elanInstanceName
1553      *            the elan instance name
1554      * @param dpId
1555      *            the dp id
1556      */
1557     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1558             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1559         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1560                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1561         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1562         if (dpnInterfaces.isPresent()) {
1563             tx.delete(dpnInterfacesId);
1564         }
1565     }
1566
1567     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1568             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1569         List<String> interfaceNames = new ArrayList<>();
1570         interfaceNames.add(interfaceName);
1571         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1572                 .withKey(new DpnInterfacesKey(dpId)).build();
1573         return dpnInterface;
1574     }
1575
1576     private static void createElanStateList(String elanInstanceName, String interfaceName,
1577             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1578         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1579         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1580         // Adding new Elan Interface Port to the operational DataStore without
1581         // Static-Mac Entries..
1582         if (elanInterfaceLists.isPresent()) {
1583             List<String> interfaceLists = new ArrayList<>();
1584             interfaceLists.add(interfaceName);
1585             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1586             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1587                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1588             }
1589
1590             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1591                     .withKey(new ElanKey(elanInstanceName)).build();
1592             tx.mergeParentStructurePut(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState);
1593             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1594                     interfaceLists);
1595         }
1596     }
1597
1598     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1599         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1600     }
1601
1602     @SuppressWarnings("checkstyle:IllegalCatch")
1603     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1604         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1605         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1606                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1607         if (dpnInterfaceLists == null) {
1608             return;
1609         }
1610         Map<ElanDpnInterfacesListKey, ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists
1611                 .nonnullElanDpnInterfacesList();
1612         for (ElanDpnInterfacesList elanDpns : elanDpnIf.values()) {
1613             int cnt = 0;
1614             String elanName = elanDpns.getElanInstanceName();
1615             ElanInstance elanInfo = elanInstanceCache.get(elanName).orElse(null);
1616             if (elanInfo == null) {
1617                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1618                         + "skipping this ELAN for tunnel handling", elanName);
1619                 continue;
1620             }
1621             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1622                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1623                 continue;
1624             }
1625             Map<DpnInterfacesKey, DpnInterfaces> dpnInterfaces = elanDpns.nonnullDpnInterfaces();
1626             if (dpnInterfaces == null) {
1627                 continue;
1628             }
1629             DpnInterfaces dstDpnIf = null;
1630             for (DpnInterfaces dpnIf : dpnInterfaces.values()) {
1631                 Uint64 dpnIfDpId = dpnIf.getDpId();
1632                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1633                     cnt++;
1634                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1635                     cnt++;
1636                     dstDpnIf = dpnIf;
1637                 }
1638             }
1639             if (cnt == 2) {
1640                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1641                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1642                 jobCoordinator.enqueueJob(elanName, () -> {
1643                     // update Remote BC Group
1644                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1645                     try {
1646                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1647                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1648                                 confTx)).get();
1649                     } catch (RuntimeException e) {
1650                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1651                     }
1652                     Set<String> interfaceLists = new HashSet<>();
1653                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1654                     for (String ifName : interfaceLists) {
1655                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1656                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1657                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1658                             if (isOperational(interfaceInfo)) {
1659                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1660                             }
1661                             return emptyList();
1662                         }, ElanConstants.JOB_MAX_RETRIES);
1663                     }
1664                     return emptyList();
1665                 }, ElanConstants.JOB_MAX_RETRIES);
1666             }
1667
1668         }
1669     }
1670
1671     /**
1672      * Handle external tunnel state event.
1673      *
1674      * @param externalTunnel
1675      *            the external tunnel
1676      * @param intrf
1677      *            the interface
1678      */
1679     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1680         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1681             return;
1682         }
1683         // dpId/externalNodeId will be available either in source or destination
1684         // based on the tunnel end point
1685         Uint64 dpId = null;
1686         NodeId externalNodeId = null;
1687         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1688             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1689             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1690         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1691             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1692             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1693         }
1694         if (dpId == null || externalNodeId == null) {
1695             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1696             return;
1697         }
1698
1699         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1700         if (dpnInterfaceLists == null) {
1701             return;
1702         }
1703         Map<ElanDpnInterfacesListKey, ElanDpnInterfacesList> elanDpnIf
1704                 = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1705         for (ElanDpnInterfacesList elanDpns : elanDpnIf.values()) {
1706             String elanName = elanDpns.getElanInstanceName();
1707             ElanInstance elanInfo = elanInstanceCache.get(elanName).orElse(null);
1708
1709             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1710             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1711                     || dpnInterfaces.getInterfaces().isEmpty()) {
1712                 continue;
1713             }
1714             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1715
1716             final Uint64 finalDpId = dpId;
1717             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1718                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1719                 "Error setting up ELAN BGs");
1720             // install L2gwDevices local macs in dpn.
1721             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1722             // Install dpn macs on external device
1723             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1724                     externalNodeId);
1725         }
1726         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1727     }
1728
1729     /**
1730      * Installs dpn macs in external device. first it checks if the physical
1731      * locator towards this dpn tep is present or not if the physical locator is
1732      * present go ahead and add the ucast macs otherwise update the mcast mac
1733      * entry to include this dpn tep ip and schedule the job to put ucast macs
1734      * once the physical locator is programmed in device
1735      *
1736      * @param elanName
1737      *            the elan name
1738      * @param lstElanInterfaceNames
1739      *            the lst Elan interface names
1740      * @param dpnId
1741      *            the dpn id
1742      * @param externalNodeId
1743      *            the external node id
1744      */
1745     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1746             NodeId externalNodeId) {
1747         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1748                 externalNodeId.getValue());
1749         if (elanL2GwDevice == null) {
1750             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1751             return;
1752         }
1753         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1754         if (dpnTepIp == null) {
1755             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1756                     externalNodeId);
1757             return;
1758         }
1759
1760         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1761         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1762                 LogicalDatastoreType.OPERATIONAL);
1763         boolean phyLocAlreadyExists =
1764                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1765                 dpnTepIp);
1766         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1767                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1768         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1769
1770         if (phyLocAlreadyExists) {
1771             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1772             return;
1773         }
1774         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1775         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1776     }
1777
1778     /**
1779      * Validate external tunnel state event.
1780      *
1781      * @param externalTunnel
1782      *            the external tunnel
1783      * @param intrf
1784      *            the intrf
1785      * @return true, if successful
1786      */
1787     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1788         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1789             String srcDevice = externalTunnel.getDestinationDevice();
1790             String destDevice = externalTunnel.getSourceDevice();
1791             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1792                     LogicalDatastoreType.CONFIGURATION);
1793             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1794                     otherEndPointExtTunnel);
1795             if (otherEndPointExtTunnel != null) {
1796                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1797                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1798                 if (otherEndPointInterfaceOperational) {
1799                     return true;
1800                 } else {
1801                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1802                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1803                 }
1804             }
1805         }
1806         return false;
1807     }
1808
1809     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1810         List<MatchInfo> mkMatches = new ArrayList<>();
1811         // Matching metadata
1812         mkMatches.add(
1813                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1814         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1815         return mkMatches;
1816     }
1817
1818     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1819                                              Uint64 dpId) {
1820         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1821         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1822             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1823                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1824         try {
1825             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1826         } catch (InterruptedException e) {
1827             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1828         }
1829     }
1830 }