elanmanager/impl/src/main/java/org/opendaylight/netvirt/elan/internal/ElanInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.elan.internal;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.infrautils.utils.concurrent.LoggingFutures.addErrorLogging;
  15 import static org.opendaylight.netvirt.elan.utils.ElanUtils.isVxlanNetworkOrVxlanSegment;
  16
  17 import com.google.common.base.Optional;
  18 import com.google.common.base.Preconditions;
  19 import com.google.common.collect.Lists;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import java.math.BigInteger;
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashSet;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.Queue;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import javax.annotation.PostConstruct;
  35 import javax.inject.Inject;
  36 import javax.inject.Singleton;
  37 import org.apache.commons.lang3.StringUtils;
  38 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  39 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  41 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  42 import org.opendaylight.genius.infra.Datastore.Configuration;
  43 import org.opendaylight.genius.infra.Datastore.Operational;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.infra.TransactionAdapter;
  47 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  48 import org.opendaylight.genius.infra.TypedWriteTransaction;
  49 import org.opendaylight.genius.interfacemanager.globals.InterfaceInfo;
  50 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  51 import org.opendaylight.genius.itm.globals.ITMConstants;
  52 import org.opendaylight.genius.mdsalutil.FlowEntity;
  53 import org.opendaylight.genius.mdsalutil.FlowEntityBuilder;
  54 import org.opendaylight.genius.mdsalutil.InstructionInfo;
  55 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  56 import org.opendaylight.genius.mdsalutil.MatchInfo;
  57 import org.opendaylight.genius.mdsalutil.MatchInfoBase;
  58 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  59 import org.opendaylight.genius.mdsalutil.NwConstants;
  60 import org.opendaylight.genius.mdsalutil.actions.ActionDrop;
  61 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  62 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  63 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  64 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  65 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  66 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteActions;
  67 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  68 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  69 import org.opendaylight.genius.mdsalutil.matches.MatchMetadata;
  70 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  71 import org.opendaylight.genius.utils.JvmGlobalLocks;
  72 import org.opendaylight.genius.utils.ServiceIndex;
  73 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  74 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  75 import org.opendaylight.infrautils.utils.concurrent.NamedSimpleReentrantLock.Acquired;
  76 import org.opendaylight.netvirt.elan.cache.ElanInstanceCache;
  77 import org.opendaylight.netvirt.elan.cache.ElanInterfaceCache;
  78 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayMulticastUtils;
  79 import org.opendaylight.netvirt.elan.l2gw.utils.ElanL2GatewayUtils;
  80 import org.opendaylight.netvirt.elan.recovery.impl.ElanServiceRecoveryHandler;
  81 import org.opendaylight.netvirt.elan.utils.ElanConstants;
  82 import org.opendaylight.netvirt.elan.utils.ElanEtreeUtils;
  83 import org.opendaylight.netvirt.elan.utils.ElanForwardingEntriesHandler;
  84 import org.opendaylight.netvirt.elan.utils.ElanItmUtils;
  85 import org.opendaylight.netvirt.elan.utils.ElanUtils;
  86 import org.opendaylight.netvirt.elanmanager.api.ElanHelper;
  87 import org.opendaylight.netvirt.elanmanager.utils.ElanL2GwCacheUtils;
  88 import org.opendaylight.netvirt.neutronvpn.api.l2gw.L2GatewayDevice;
  89 import org.opendaylight.netvirt.neutronvpn.api.utils.NeutronUtils;
  90 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  91 import org.opendaylight.serviceutils.srm.RecoverableListener;
  92 import org.opendaylight.serviceutils.srm.ServiceRecoveryRegistry;
  93 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  94 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  95 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.servicebinding.rev160406.service.bindings.services.info.BoundServices;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.external.tunnel.list.ExternalTunnel;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.GroupKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInstance;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeInterface.EtreeInterfaceType;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.etree.rev160614.EtreeLeafTagName;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanDpnInterfaces;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanForwardingTables;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInterfaces;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMac;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan._interface.forwarding.entries.ElanInterfaceMacKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.ElanDpnInterfacesList;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfacesKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTable;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.forwarding.tables.MacTableKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.elan._interface.StaticMacEntries;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.Elan;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanBuilder;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.state.ElanKey;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntry;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryBuilder;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.forwarding.entries.MacEntryKey;
 139 import org.opendaylight.yang.gen.v1.urn.opendaylight.neutron.ports.rev150712.ports.attributes.ports.Port;
 140 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg1;
 141 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.ovsdb.hwvtep.rev150901.hwvtep.global.attributes.RemoteMcastMacs;
 142 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
 143 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 144 import org.opendaylight.yangtools.yang.common.Uint32;
 145 import org.opendaylight.yangtools.yang.common.Uint64;
 146 import org.slf4j.Logger;
 147 import org.slf4j.LoggerFactory;
 148
 149 /**
 150  * Class in charge of handling creations, modifications and removals of
 151  * ElanInterfaces.
 152  *
 153  * @see org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.interfaces.ElanInterface
 154  */
 155 @Singleton
 156 public class ElanInterfaceManager extends AsyncDataTreeChangeListenerBase<ElanInterface, ElanInterfaceManager>
 157         implements RecoverableListener {
 158     private static final Logger LOG = LoggerFactory.getLogger(ElanInterfaceManager.class);
 159     private static final Logger EVENT_LOGGER = LoggerFactory.getLogger("NetvirtEventLogger");
 160     public static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 161     private static final boolean SH_FLAG_SET = true;
 162     private static final boolean SH_FLAG_UNSET = false;
 163
 164     private final DataBroker broker;
 165     private final ManagedNewTransactionRunner txRunner;
 166     private final IMdsalApiManager mdsalManager;
 167     private final IInterfaceManager interfaceManager;
 168     private final IdManagerService idManager;
 169     private final ElanForwardingEntriesHandler elanForwardingEntriesHandler;
 170     private final INeutronVpnManager neutronVpnManager;
 171     private final ElanItmUtils elanItmUtils;
 172     private final ElanEtreeUtils elanEtreeUtils;
 173     private final ElanL2GatewayUtils elanL2GatewayUtils;
 174     private final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils;
 175     private final ElanUtils elanUtils;
 176     private final JobCoordinator jobCoordinator;
 177     private final ElanInstanceCache elanInstanceCache;
 178     private final ElanInterfaceCache elanInterfaceCache;
 179     private final ElanGroupCache elanGroupCache;
 180
 181     private final Map<String, ConcurrentLinkedQueue<ElanInterface>>
 182         unProcessedElanInterfaces = new ConcurrentHashMap<>();
 183
 184     @Inject
 185     public ElanInterfaceManager(final DataBroker dataBroker, final IdManagerService managerService,
 186                                 final IMdsalApiManager mdsalApiManager, IInterfaceManager interfaceManager,
 187                                 final ElanForwardingEntriesHandler elanForwardingEntriesHandler,
 188                                 final INeutronVpnManager neutronVpnManager, final ElanItmUtils elanItmUtils,
 189                                 final ElanEtreeUtils elanEtreeUtils, final ElanL2GatewayUtils elanL2GatewayUtils,
 190                                 final ElanUtils elanUtils, final JobCoordinator jobCoordinator,
 191                                 final ElanL2GatewayMulticastUtils elanL2GatewayMulticastUtils,
 192                                 final ElanInstanceCache elanInstanceCache,
 193                                 final ElanInterfaceCache elanInterfaceCache,
 194                                 final ElanServiceRecoveryHandler elanServiceRecoveryHandler,
 195                                 ElanGroupCache elanGroupCache,
 196                                 final ServiceRecoveryRegistry serviceRecoveryRegistry) {
 197         super(ElanInterface.class, ElanInterfaceManager.class);
 198         this.broker = dataBroker;
 199         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 200         this.idManager = managerService;
 201         this.mdsalManager = mdsalApiManager;
 202         this.interfaceManager = interfaceManager;
 203         this.elanForwardingEntriesHandler = elanForwardingEntriesHandler;
 204         this.neutronVpnManager = neutronVpnManager;
 205         this.elanItmUtils = elanItmUtils;
 206         this.elanEtreeUtils = elanEtreeUtils;
 207         this.elanL2GatewayUtils = elanL2GatewayUtils;
 208         this.elanUtils = elanUtils;
 209         this.jobCoordinator = jobCoordinator;
 210         this.elanL2GatewayMulticastUtils = elanL2GatewayMulticastUtils;
 211         this.elanInstanceCache = elanInstanceCache;
 212         this.elanInterfaceCache = elanInterfaceCache;
 213         this.elanGroupCache = elanGroupCache;
 214         serviceRecoveryRegistry.addRecoverableListener(elanServiceRecoveryHandler.buildServiceRegistryKey(), this);
 215     }
 216
 217     @Override
 218     @PostConstruct
 219     public void init() {
 220         registerListener();
 221     }
 222
 223     @Override
 224     public void registerListener() {
 225         registerListener(LogicalDatastoreType.CONFIGURATION, broker);
 226     }
 227
 228     @Override
 229     protected InstanceIdentifier<ElanInterface> getWildCardPath() {
 230         return InstanceIdentifier.create(ElanInterfaces.class).child(ElanInterface.class);
 231     }
 232
 233     @Override
 234     protected void remove(InstanceIdentifier<ElanInterface> identifier, ElanInterface del) {
 235         String interfaceName = del.getName();
 236         String elanInstanceName = del.getElanInstanceName();
 237         EVENT_LOGGER.debug("ELAN-Interface, REMOVE {} Instance {}", interfaceName, elanInstanceName);
 238         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 239         if (elanInterfaces != null && elanInterfaces.contains(del)) {
 240             elanInterfaces.remove(del);
 241             if (elanInterfaces.isEmpty()) {
 242                 unProcessedElanInterfaces.remove(elanInstanceName);
 243             }
 244         }
 245         ElanInstance elanInfo = elanInstanceCache.get(elanInstanceName).orNull();
 246         /*
 247          * Handling in case the elan instance is deleted.If the Elan instance is
 248          * deleted, there is no need to explicitly delete the elan interfaces
 249          */
 250         if (elanInfo == null) {
 251             return;
 252         }
 253         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 254         if (interfaceInfo == null && elanInfo.isExternal()) {
 255             // In deleting external network, the underlying ietf Inteface might have been removed
 256             // from the config DS prior to deleting the ELAN interface. We try to get the InterfaceInfo
 257             // from Operational DS instead
 258             interfaceInfo = interfaceManager.getInterfaceInfoFromOperationalDataStore(interfaceName);
 259         }
 260         InterfaceRemoveWorkerOnElan configWorker = new InterfaceRemoveWorkerOnElan(elanInstanceName, elanInfo,
 261                 interfaceName, interfaceInfo, this);
 262         jobCoordinator.enqueueJob(elanInstanceName, configWorker, ElanConstants.JOB_MAX_RETRIES);
 263     }
 264
 265     private static class RemoveElanInterfaceHolder {
 266         boolean isLastElanInterface = false;
 267         Uint64 dpId = null;
 268     }
 269
 270     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 271     public List<ListenableFuture<Void>> removeElanInterface(ElanInstance elanInfo, String interfaceName,
 272             InterfaceInfo interfaceInfo) {
 273         String elanName = elanInfo.getElanInstanceName();
 274         EVENT_LOGGER.debug("ELAN-InterfaceState, REMOVE {} Instance {}", interfaceName, elanName);
 275         Uint32 elanTag = elanInfo.getElanTag();
 276         // We use two transaction so we don't suffer on multiple shards (interfaces and flows)
 277         List<ListenableFuture<Void>> futures = new ArrayList<>();
 278         RemoveElanInterfaceHolder holder = new RemoveElanInterfaceHolder();
 279         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 280             Elan elanState = removeElanStateForInterface(elanInfo, interfaceName, interfaceTx);
 281             if (elanState == null) {
 282                 return;
 283             }
 284             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 285                 List<String> elanInterfaces = elanState.getElanInterfaces();
 286                 if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 287                     holder.isLastElanInterface = true;
 288                 }
 289                 if (interfaceInfo != null) {
 290                     holder.dpId = interfaceInfo.getDpId();
 291                     DpnInterfaces dpnInterfaces = removeElanDpnInterfaceFromOperationalDataStore(elanName, holder.dpId,
 292                         interfaceName, elanTag, interfaceTx);
 293                     /*
 294                      * If there are not elan ports, remove the unknown dmac, terminating
 295                      * service table flows, remote/local bc group
 296                      */
 297                     if (dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
 298                         || dpnInterfaces.getInterfaces().isEmpty()) {
 299                         // No more Elan Interfaces in this DPN
 300                         EVENT_LOGGER.debug("ELAN-Flows, REMOVE {} Instance {}", interfaceName, elanName);
 301                         LOG.debug("deleting the elan: {} present on dpId: {}", elanInfo.getElanInstanceName(),
 302                             holder.dpId);
 303                         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 304                             removeDefaultTermFlow(holder.dpId, elanInfo.getElanTag().toJava());
 305                         }
 306                         removeUnknownDmacFlow(holder.dpId, elanInfo, flowTx, elanInfo.getElanTag().toJava());
 307                         removeEtreeUnknownDmacFlow(holder.dpId, elanInfo, flowTx);
 308                         removeElanBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 309                         removeLocalBroadcastGroup(elanInfo, interfaceInfo, flowTx);
 310                         removeEtreeBroadcastGrups(elanInfo, interfaceInfo, flowTx);
 311                         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
 312                             if (elanUtils.isOpenstackVniSemanticsEnforced()) {
 313                                 elanUtils.removeTerminatingServiceAction(holder.dpId,
 314                                     ElanUtils.getVxlanSegmentationId(elanInfo).intValue());
 315                             }
 316                             unsetExternalTunnelTable(holder.dpId, elanInfo, flowTx);
 317                         }
 318                     } else {
 319                         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, flowTx);
 320                     }
 321                 }
 322             }));
 323         }));
 324         futures.forEach(ElanUtils::waitForTransactionToComplete);
 325
 326         InterfaceRemoveWorkerOnElanInterface removeInterfaceWorker = new InterfaceRemoveWorkerOnElanInterface(
 327                 interfaceName, elanInfo, interfaceInfo, this, holder.isLastElanInterface);
 328         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(interfaceName), removeInterfaceWorker,
 329                 ElanConstants.JOB_MAX_RETRIES);
 330
 331         return futures;
 332     }
 333
 334     private void removeEtreeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
 335             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 336             throws ExecutionException, InterruptedException {
 337         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanInfo.getElanTag().toJava());
 338         if (etreeLeafTag != null) {
 339             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
 340             removeUnknownDmacFlow(dpId, elanInfo, deleteFlowGroupTx, leafTag);
 341         }
 342     }
 343
 344     private void removeEtreeBroadcastGrups(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 345             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 346             throws ExecutionException, InterruptedException {
 347         removeLeavesEtreeBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 348         removeLeavesLocalBroadcastGroup(elanInfo, interfaceInfo, deleteFlowGroupTx);
 349     }
 350
 351     private void removeLeavesLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 352             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 353             throws ExecutionException, InterruptedException {
 354         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 355         if (etreeInstance != null) {
 356             Uint64 dpnId = interfaceInfo.getDpId();
 357             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeInstance.getEtreeLeafTagVal().getValue().toJava());
 358             LOG.trace("deleted the localBroadCast Group:{}", groupId);
 359             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 360         }
 361     }
 362
 363     private void removeLeavesEtreeBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 364             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
 365             throws ExecutionException, InterruptedException {
 366         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
 367         if (etreeInstance != null) {
 368             long etreeTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
 369             Uint64 dpnId = interfaceInfo.getDpId();
 370             long groupId = ElanUtils.getEtreeLeafRemoteBCGId(etreeTag);
 371             LOG.trace("deleting the remoteBroadCast group:{}", groupId);
 372             mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
 373         }
 374     }
 375
 376     private static Elan removeElanStateForInterface(ElanInstance elanInfo, String interfaceName,
 377             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
 378         String elanName = elanInfo.getElanInstanceName();
 379         Elan elanState = ElanUtils.getElanByName(tx, elanName);
 380         if (elanState == null) {
 381             return elanState;
 382         }
 383         List<String> existingElanInterfaces = elanState.getElanInterfaces();
 384         List<String> elanInterfaces = new ArrayList<>();
 385         if (existingElanInterfaces != null) {
 386             elanInterfaces.addAll(existingElanInterfaces);
 387         }
 388         boolean isRemoved = elanInterfaces.remove(interfaceName);
 389         if (!isRemoved) {
 390             return elanState;
 391         }
 392
 393         if (elanInterfaces.isEmpty()) {
 394             tx.delete(ElanUtils.getElanInstanceOperationalDataPath(elanName));
 395             tx.delete(ElanUtils.getElanMacTableOperationalDataPath(elanName));
 396             tx.delete(ElanUtils.getElanInfoEntriesOperationalDataPath(elanInfo.getElanTag()));
 397             tx.delete(ElanUtils.getElanDpnOperationDataPath(elanName));
 398         } else {
 399             Elan updateElanState = new ElanBuilder().setElanInterfaces(elanInterfaces).setName(elanName)
 400                     .withKey(new ElanKey(elanName)).build();
 401             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanName), updateElanState);
 402         }
 403         return elanState;
 404     }
 405
 406     private void deleteElanInterfaceFromConfigDS(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
 407             throws ReadFailedException {
 408         // removing the ElanInterface from the config data_store if interface is
 409         // not present in Interface config DS
 410         if (interfaceManager.getInterfaceInfoFromConfigDataStore(TransactionAdapter.toReadWriteTransaction(tx),
 411             interfaceName) == null
 412                 && elanInterfaceCache.get(interfaceName).isPresent()) {
 413             tx.delete(ElanUtils.getElanInterfaceConfigurationDataPathId(interfaceName));
 414         }
 415     }
 416
 417     List<ListenableFuture<Void>> removeEntriesForElanInterface(ElanInstance elanInfo, InterfaceInfo
 418             interfaceInfo, String interfaceName, boolean isLastElanInterface) {
 419         String elanName = elanInfo.getElanInstanceName();
 420         EVENT_LOGGER.debug("ELAN-InterfaceEntries, REMOVE {} Instance {}", interfaceName, elanName);
 421         List<ListenableFuture<Void>> futures = new ArrayList<>();
 422         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, flowTx -> {
 423             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, interfaceTx -> {
 424                 InstanceIdentifier<ElanInterfaceMac> elanInterfaceId = ElanUtils
 425                         .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
 426                 Optional<ElanInterfaceMac> existingElanInterfaceMac = interfaceTx.read(elanInterfaceId).get();
 427                 LOG.debug("Removing the Interface:{} from elan:{}", interfaceName, elanName);
 428                 if (interfaceInfo != null) {
 429                     if (existingElanInterfaceMac.isPresent()) {
 430                         List<MacEntry> existingMacEntries = existingElanInterfaceMac.get().getMacEntry();
 431                         if (existingMacEntries != null) {
 432                             List<PhysAddress> macAddresses = new ArrayList<>();
 433                             for (MacEntry macEntry : existingMacEntries) {
 434                                 PhysAddress macAddress = macEntry.getMacAddress();
 435                                 LOG.debug("removing the  mac-entry:{} present on elanInterface:{}",
 436                                         macAddress.getValue(), interfaceName);
 437                                 Optional<MacEntry> macEntryOptional =
 438                                         elanUtils.getMacEntryForElanInstance(interfaceTx, elanName, macAddress);
 439                                 if (!isLastElanInterface && macEntryOptional.isPresent()) {
 440                                     interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 441                                 }
 442                                 EVENT_LOGGER.debug("ELAN-MacFlows, REMOVE {} Instance {} Mac {}",
 443                                         interfaceName, elanName, macAddress);
 444                                 elanUtils.deleteMacFlows(elanInfo, interfaceInfo, macEntry, flowTx);
 445                                 macAddresses.add(macAddress);
 446                             }
 447
 448                             // Removing all those MACs from External Devices belonging
 449                             // to this ELAN
 450                             if (isVxlanNetworkOrVxlanSegment(elanInfo) && ! macAddresses.isEmpty()) {
 451                                 elanL2GatewayUtils.removeMacsFromElanExternalDevices(elanInfo, macAddresses);
 452                             }
 453                         }
 454                     }
 455                     removeDefaultTermFlow(interfaceInfo.getDpId(), interfaceInfo.getInterfaceTag());
 456                     removeFilterEqualsTable(elanInfo, interfaceInfo, flowTx);
 457                 } else if (existingElanInterfaceMac.isPresent()) {
 458                     // Interface does not exist in ConfigDS, so lets remove everything
 459                     // about that interface related to Elan
 460                     List<MacEntry> macEntries = existingElanInterfaceMac.get().getMacEntry();
 461                     if (macEntries != null) {
 462                         for (MacEntry macEntry : macEntries) {
 463                             PhysAddress macAddress = macEntry.getMacAddress();
 464                             if (elanUtils.getMacEntryForElanInstance(elanName, macAddress).isPresent()) {
 465                                 interfaceTx.delete(ElanUtils.getMacEntryOperationalDataPath(elanName, macAddress));
 466                             }
 467                         }
 468                     }
 469                 }
 470                 if (existingElanInterfaceMac.isPresent()) {
 471                     interfaceTx.delete(elanInterfaceId);
 472                 }
 473                 unbindService(interfaceName, flowTx);
 474                 deleteElanInterfaceFromConfigDS(interfaceName, flowTx);
 475             }));
 476         }));
 477         return futures;
 478     }
 479
 480     private DpnInterfaces removeElanDpnInterfaceFromOperationalDataStore(String elanName, Uint64 dpId,
 481                                                                          String interfaceName, Uint32 elanTag,
 482                                                                          TypedReadWriteTransaction<Operational> tx)
 483             throws ExecutionException, InterruptedException {
 484         // FIXME: pass in and use ElanInstanceKey instead?
 485         final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanName);
 486         lock.lock();
 487         try {
 488             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
 489             if (dpnInterfaces != null) {
 490                 List<String> interfaceLists = null;
 491                 if (dpnInterfaces.getInterfaces() != null) {
 492                     interfaceLists = new ArrayList<>(dpnInterfaces.getInterfaces());
 493                 }
 494                 if (interfaceLists != null) {
 495                     interfaceLists.remove(interfaceName);
 496                 }
 497
 498                 if (interfaceLists == null || interfaceLists.isEmpty()) {
 499                     deleteAllRemoteMacsInADpn(elanName, dpId, elanTag);
 500                     deleteElanDpnInterface(elanName, dpId, tx);
 501                 } else {
 502                     dpnInterfaces = updateElanDpnInterfacesList(elanName, dpId, interfaceLists, tx);
 503                 }
 504             }
 505             return dpnInterfaces;
 506         } finally {
 507             lock.unlock();
 508         }
 509     }
 510
 511     private void deleteAllRemoteMacsInADpn(String elanName, Uint64 dpId, Uint32 elanTag) {
 512         List<DpnInterfaces> dpnInterfaces = elanUtils.getInvolvedDpnsInElan(elanName);
 513         addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 514             for (DpnInterfaces dpnInterface : dpnInterfaces) {
 515                 Uint64 currentDpId = dpnInterface.getDpId();
 516                 if (!currentDpId.equals(dpId) && dpnInterface.getInterfaces() != null) {
 517                     for (String elanInterface : dpnInterface.getInterfaces()) {
 518                         ElanInterfaceMac macs = elanUtils.getElanInterfaceMacByInterfaceName(elanInterface);
 519                         if (macs == null || macs.getMacEntry() == null) {
 520                             continue;
 521                         }
 522                         for (MacEntry mac : macs.getMacEntry()) {
 523                             removeTheMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 524                             removeEtreeMacFlowInTheDPN(dpId, elanTag, currentDpId, mac, confTx);
 525                         }
 526                     }
 527                 }
 528             }
 529         }), LOG, "Error deleting remote MACs in DPN {}", dpId);
 530     }
 531
 532     private void removeEtreeMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 533             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 534         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag.longValue());
 535         if (etreeLeafTag != null) {
 536             removeTheMacFlowInTheDPN(dpId, etreeLeafTag.getEtreeLeafTag().getValue(),
 537                 currentDpId, mac, confTx);
 538         }
 539     }
 540
 541     private void removeTheMacFlowInTheDPN(Uint64 dpId, Uint32 elanTag, Uint64 currentDpId, MacEntry mac,
 542             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 543         mdsalManager
 544                 .removeFlow(confTx, dpId,
 545                         MDSALUtil.buildFlow(NwConstants.ELAN_DMAC_TABLE,
 546                                 ElanUtils.getKnownDynamicmacFlowRef(NwConstants.ELAN_DMAC_TABLE, dpId, currentDpId,
 547                                         mac.getMacAddress().getValue(), elanTag)));
 548     }
 549
 550     /*
 551     * Possible Scenarios for update
 552     *   a. if orig={1,2,3,4}   and updated=null or updated={}
 553         then all {1,2,3,4} should be removed
 554
 555         b. if orig=null or orig={}  and  updated ={1,2,3,4}
 556         then all {1,2,3,4} should be added
 557
 558         c. if orig = {1,2,3,4} updated={2,3,4}
 559         then 1 should be removed
 560
 561         d. basically if orig = { 1,2,3,4} and updated is {1,2,3,4,5}
 562         then we should just add 5
 563
 564         e. if orig = {1,2,3,4} updated={2,3,4,5}
 565         then 1 should be removed , 5 should be added
 566     * */
 567     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 568     @Override
 569     protected void update(InstanceIdentifier<ElanInterface> identifier, ElanInterface original, ElanInterface update) {
 570         // updating the static-Mac Entries for the existing elanInterface
 571         String elanName = update.getElanInstanceName();
 572         String interfaceName = update.getName();
 573         LOG.info("Update static mac entries for elan interface {} in elan instance {}", interfaceName, elanName);
 574         EVENT_LOGGER.debug("ELAN-Interface, UPDATE {} Instance {}", original.getName(), elanName);
 575
 576         List<StaticMacEntries> originalStaticMacEntries = original.getStaticMacEntries();
 577         List<StaticMacEntries> updatedStaticMacEntries = update.getStaticMacEntries();
 578         List<StaticMacEntries> deletedEntries = ElanUtils.diffOf(originalStaticMacEntries, updatedStaticMacEntries);
 579         List<StaticMacEntries> updatedEntries = ElanUtils.diffOf(updatedStaticMacEntries, originalStaticMacEntries);
 580
 581         deletedEntries.forEach((deletedEntry) -> removeInterfaceStaticMacEntries(elanName, interfaceName,
 582                 deletedEntry.getMacAddress()));
 583
 584         /*if updatedStaticMacEntries is NOT NULL, which means as part of update call these entries were added.
 585         * Hence add the macentries for the same.*/
 586         for (StaticMacEntries staticMacEntry : updatedEntries) {
 587             InstanceIdentifier<MacEntry> macEntryIdentifier = getMacEntryOperationalDataPath(elanName,
 588                     staticMacEntry.getMacAddress());
 589             addErrorLogging(ElanUtils.waitForTransactionToComplete(
 590                 txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 591                     Optional<MacEntry> existingMacEntry = tx.read(macEntryIdentifier).get();
 592                     if (existingMacEntry.isPresent()) {
 593                         LOG.debug("updating elan interface forwarding table for mac entry {} elan instance {}",
 594                                 existingMacEntry.get(), elanName);
 595                         elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 596                             elanName, interfaceName, existingMacEntry.get().getInterface(), existingMacEntry.get(),
 597                             tx);
 598                     } else {
 599                         LOG.info("Adding elan interface forwarding table for mac entry {} elan interface"
 600                                 + " {} elan instance {}.", staticMacEntry.getMacAddress(), interfaceName, elanName);
 601                         elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(
 602                             elanName, interfaceName, staticMacEntry, tx);
 603                     }
 604                 })), LOG, "Error in update: identifier={}, original={}, update={}", identifier, original, update);
 605         }
 606     }
 607
 608     @Override
 609     protected void add(InstanceIdentifier<ElanInterface> identifier, ElanInterface elanInterfaceAdded) {
 610         LOG.info("Init for ELAN interface Add {}", elanInterfaceAdded);
 611         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
 612             String elanInstanceName = elanInterfaceAdded.getElanInstanceName();
 613             String interfaceName = elanInterfaceAdded.getName();
 614             EVENT_LOGGER.debug("ELAN-Interface, ADD {} Instance {}", interfaceName, elanInstanceName);
 615             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 616             if (interfaceInfo == null) {
 617                 LOG.info("Interface {} is removed from Interface Oper DS due to port down ", interfaceName);
 618                 return;
 619             }
 620             ElanInstance elanInstance = elanInstanceCache.get(elanInstanceName).orNull();
 621
 622             if (elanInstance == null) {
 623                 // Add the ElanInstance in the Configuration data-store
 624                 List<String> elanInterfaces = new ArrayList<>();
 625                 elanInterfaces.add(interfaceName);
 626                 elanInstance = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 627                     confTx -> ElanUtils.updateOperationalDataStore(idManager,
 628                         new ElanInstanceBuilder().setElanInstanceName(elanInstanceName).setDescription(
 629                             elanInterfaceAdded.getDescription()).build(), elanInterfaces, confTx, operTx)).get();
 630             }
 631
 632             Uint32 elanTag = elanInstance.getElanTag();
 633             // If elan tag is not updated, then put the elan interface into
 634             // unprocessed entry map and entry. Let entries
 635             // in this map get processed during ELAN update DCN.
 636             if (elanTag == null || elanTag.longValue() == 0L) {
 637                 ConcurrentLinkedQueue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstanceName);
 638                 if (elanInterfaces == null) {
 639                     elanInterfaces = new ConcurrentLinkedQueue<>();
 640                 }
 641                 if (!elanInterfaces.contains(elanInterfaceAdded)) {
 642                     elanInterfaces.add(elanInterfaceAdded);
 643                 }
 644                 unProcessedElanInterfaces.put(elanInstanceName, elanInterfaces);
 645                 return;
 646             }
 647             InterfaceAddWorkerOnElan addWorker = new InterfaceAddWorkerOnElan(elanInstanceName, elanInterfaceAdded,
 648                     interfaceInfo, elanInstance, this);
 649             jobCoordinator.enqueueJob(elanInstanceName, addWorker, ElanConstants.JOB_MAX_RETRIES);
 650         }), LOG, "Error processing added ELAN interface");
 651     }
 652
 653     List<ListenableFuture<Void>> handleunprocessedElanInterfaces(ElanInstance elanInstance) {
 654         LOG.trace("Handling unprocessed elan interfaces for elan instance {}", elanInstance.getElanInstanceName());
 655         List<ListenableFuture<Void>> futures = new ArrayList<>();
 656         Queue<ElanInterface> elanInterfaces = unProcessedElanInterfaces.get(elanInstance.getElanInstanceName());
 657         if (elanInterfaces == null || elanInterfaces.isEmpty()) {
 658             return futures;
 659         }
 660         for (ElanInterface elanInterface : elanInterfaces) {
 661             String interfaceName = elanInterface.getName();
 662             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 663             futures.addAll(addElanInterface(elanInterface, interfaceInfo, elanInstance));
 664         }
 665         unProcessedElanInterfaces.remove(elanInstance.getElanInstanceName());
 666         LOG.info("Unprocessed elan interfaces for elan instance {} has been removed.",
 667                 elanInstance.getElanInstanceName());
 668         return futures;
 669     }
 670
 671     void programRemoteDmacFlow(ElanInstance elanInstance, InterfaceInfo interfaceInfo,
 672             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
 673         ElanDpnInterfacesList elanDpnInterfacesList = elanUtils
 674                 .getElanDpnInterfacesList(elanInstance.getElanInstanceName());
 675         List<DpnInterfaces> dpnInterfaceLists = null;
 676         if (elanDpnInterfacesList != null) {
 677             dpnInterfaceLists = elanDpnInterfacesList.getDpnInterfaces();
 678         }
 679         if (dpnInterfaceLists == null) {
 680             dpnInterfaceLists = new ArrayList<>();
 681         }
 682         for (DpnInterfaces dpnInterfaces : dpnInterfaceLists) {
 683             Uint64 dstDpId = interfaceInfo.getDpId();
 684             if (Objects.equals(dpnInterfaces.getDpId(), dstDpId)) {
 685                 continue;
 686             }
 687             List<String> remoteElanInterfaces = dpnInterfaces.getInterfaces();
 688             for (String remoteIf : remoteElanInterfaces) {
 689                 ElanInterfaceMac elanIfMac = elanUtils.getElanInterfaceMacByInterfaceName(remoteIf);
 690                 InterfaceInfo remoteInterface = interfaceManager.getInterfaceInfo(remoteIf);
 691                 if (elanIfMac == null || remoteInterface == null) {
 692                     continue;
 693                 }
 694                 List<MacEntry> remoteMacEntries = elanIfMac.getMacEntry();
 695                 if (remoteMacEntries != null) {
 696                     for (MacEntry macEntry : remoteMacEntries) {
 697                         String macAddress = macEntry.getMacAddress().getValue();
 698                         LOG.info("Programming remote dmac {} on the newly added DPN {} for elan {}", macAddress,
 699                                 dstDpId, elanInstance.getElanInstanceName());
 700                         elanUtils.setupRemoteDmacFlow(dstDpId, remoteInterface.getDpId(),
 701                                 remoteInterface.getInterfaceTag(), elanInstance.getElanTag(), macAddress,
 702                                 elanInstance.getElanInstanceName(), writeFlowGroupTx, remoteIf, elanInstance);
 703                     }
 704                 }
 705             }
 706         }
 707     }
 708
 709     private static class AddElanInterfaceHolder {
 710         private DpnInterfaces dpnInterfaces = null;
 711         private boolean isFirstInterfaceInDpn = false;
 712         private Uint64 dpId;
 713     }
 714
 715     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 716     List<ListenableFuture<Void>> addElanInterface(ElanInterface elanInterface,
 717             InterfaceInfo interfaceInfo, ElanInstance elanInstance) {
 718         Preconditions.checkNotNull(elanInstance, "elanInstance cannot be null");
 719         Preconditions.checkNotNull(interfaceInfo, "interfaceInfo cannot be null");
 720         Preconditions.checkNotNull(elanInterface, "elanInterface cannot be null");
 721
 722         String interfaceName = elanInterface.getName();
 723         String elanInstanceName = elanInterface.getElanInstanceName();
 724         LOG.trace("Adding elan interface: interface name {} , instance name {}", interfaceName, elanInstanceName);
 725         EVENT_LOGGER.debug("ELAN-InterfaceState, ADD {} Instance {}", interfaceName, elanInstanceName);
 726
 727         List<ListenableFuture<Void>> futures = new ArrayList<>();
 728         AddElanInterfaceHolder holder = new AddElanInterfaceHolder();
 729         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 730             Elan elanInfo = ElanUtils.getElanByName(broker, elanInstanceName);
 731             if (elanInfo == null) {
 732                 LOG.trace("elanInfo is null for elan instance: {}", elanInstanceName);
 733                 List<String> elanInterfaces = new ArrayList<>();
 734                 elanInterfaces.add(interfaceName);
 735                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 736                     confTx -> ElanUtils.updateOperationalDataStore(idManager, elanInstance, elanInterfaces, confTx,
 737                         operTx)));
 738             } else {
 739                 createElanStateList(elanInstanceName, interfaceName, operTx);
 740             }
 741             // Specific actions to the DPN where the ElanInterface has been added,
 742             // for example, programming the
 743             // External tunnel table if needed or adding the ElanInterface to the
 744             // DpnInterfaces in the operational DS.
 745             holder.dpId = interfaceInfo.getDpId();
 746             if (holder.dpId != null && !holder.dpId.equals(ElanConstants.INVALID_DPN)) {
 747                 // FIXME: use elanInstaince.key() instead?
 748                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(elanInstanceName);
 749                 lock.lock();
 750                 try {
 751                     InstanceIdentifier<DpnInterfaces> elanDpnInterfaces = ElanUtils
 752                         .getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId);
 753                     Optional<DpnInterfaces> existingElanDpnInterfaces = operTx.read(elanDpnInterfaces).get();
 754                     if (ElanUtils.isVlan(elanInstance)) {
 755                         holder.isFirstInterfaceInDpn =  checkIfFirstInterface(interfaceName,
 756                             elanInstanceName, existingElanDpnInterfaces);
 757                     } else {
 758                         holder.isFirstInterfaceInDpn = !existingElanDpnInterfaces.isPresent();
 759                     }
 760                     if (holder.isFirstInterfaceInDpn) {
 761                         // ELAN's 1st ElanInterface added to this DPN
 762                         if (!existingElanDpnInterfaces.isPresent()) {
 763                             holder.dpnInterfaces =
 764                                 createElanInterfacesList(elanInstanceName, interfaceName, holder.dpId, operTx);
 765                         } else {
 766                             List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 767                             List<String> elanInterfaces =
 768                                 existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 769                             elanInterfaces.add(interfaceName);
 770                             holder.dpnInterfaces = updateElanDpnInterfacesList(elanInstanceName, holder.dpId,
 771                                 elanInterfaces, operTx);
 772                         }
 773                         LOG.debug("1st interface {} for elan {} is added to dpn {}",
 774                                 interfaceName, elanInstanceName, holder.dpId);
 775                         // The 1st ElanInterface in a DPN must program the Ext Tunnel
 776                         // table, but only if Elan has VNI
 777                         if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 778                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 779                                 confTx -> setExternalTunnelTable(holder.dpId, elanInstance, confTx)));
 780                         }
 781                         elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 782                             interfaceName);
 783                     } else {
 784                         List<String> existingInterfaces = existingElanDpnInterfaces.get().getInterfaces();
 785                         List<String> elanInterfaces =
 786                             existingInterfaces != null ? new ArrayList<>(existingInterfaces) : new ArrayList<>();
 787                         elanInterfaces.add(interfaceName);
 788                         if (elanInterfaces.size() == 1) { // 1st dpn interface
 789                             elanL2GatewayUtils.installElanL2gwDevicesLocalMacsInDpn(holder.dpId, elanInstance,
 790                                 interfaceName);
 791                         }
 792                         holder.dpnInterfaces =
 793                             updateElanDpnInterfacesList(elanInstanceName, holder.dpId, elanInterfaces, operTx);
 794                         LOG.debug("Interface {} for elan {} is added to dpn {}",
 795                                 interfaceName, elanInstanceName, holder.dpId);
 796                     }
 797                 } finally {
 798                     lock.unlock();
 799                 }
 800             }
 801
 802             // add code to install Local/Remote BC group, unknow DMAC entry,
 803             // terminating service table flow entry
 804             // call bindservice of interfacemanager to create ingress table flow
 805             // enty.
 806             // Add interface to the ElanInterfaceForwardingEntires Container
 807             createElanInterfaceTablesList(interfaceName, operTx);
 808         }));
 809         futures.forEach(ElanUtils::waitForTransactionToComplete);
 810         futures.add(
 811             ElanUtils.waitForTransactionToComplete(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 812                 confTx -> installEntriesForFirstInterfaceonDpn(elanInstance, interfaceInfo, holder.dpnInterfaces,
 813                     holder.isFirstInterfaceInDpn, confTx))));
 814
 815         // add the vlan provider interface to remote BC group for the elan
 816         // for internal vlan networks
 817         if (ElanUtils.isVlan(elanInstance) && !elanInstance.isExternal()) {
 818             if (interfaceManager.isExternalInterface(interfaceName)) {
 819                 LOG.debug("adding vlan prv intf {} to elan {} BC group", interfaceName, elanInstanceName);
 820                 handleExternalInterfaceEvent(elanInstance, holder.dpnInterfaces, holder.dpId);
 821             }
 822         }
 823         if (holder.isFirstInterfaceInDpn) {
 824             // ELAN's 1st ElanInterface added to this DPN
 825             LOG.debug("Adding dpn into operational dpn list {}", holder.dpId);
 826             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 827                 operTx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, holder.dpId),
 828                         holder.dpnInterfaces, CREATE_MISSING_PARENTS);
 829             }));
 830         } else {
 831             LOG.debug("Updated dpn into operational dpn list {}", holder.dpId);
 832         }
 833
 834         scheduleElanInterfaceWorkerAfterRemoteBcGroup(elanInstance, interfaceInfo, holder.dpnInterfaces,
 835                 holder.isFirstInterfaceInDpn, elanInterface);
 836         return futures;
 837     }
 838
 839     @SuppressWarnings("checkstyle:ForbidCertainMethod")
 840     List<ListenableFuture<Void>> setupEntriesForElanInterface(ElanInstance elanInstance,
 841             ElanInterface elanInterface, InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn) {
 842         String elanInstanceName = elanInstance.getElanInstanceName();
 843         String interfaceName = elanInterface.getName();
 844         List<ListenableFuture<Void>> futures = new ArrayList<>();
 845         Uint64 dpId = interfaceInfo.getDpId();
 846         boolean isInterfaceOperational = isOperational(interfaceInfo);
 847         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 848             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
 849                 installEntriesForElanInterface(elanInstance, elanInterface, interfaceInfo,
 850                     isFirstInterfaceInDpn, confTx);
 851
 852                 List<StaticMacEntries> staticMacEntriesList = elanInterface.getStaticMacEntries();
 853                 List<PhysAddress> staticMacAddresses = Lists.newArrayList();
 854
 855                 if (ElanUtils.isNotEmpty(staticMacEntriesList)) {
 856                     for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 857                         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName,
 858                             staticMacEntry.getMacAddress());
 859                         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 860                             LogicalDatastoreType.OPERATIONAL, macId);
 861                         if (existingMacEntry.isPresent()) {
 862                             elanForwardingEntriesHandler.updateElanInterfaceForwardingTablesList(
 863                                 elanInstanceName, interfaceName, existingMacEntry.get().getInterface(),
 864                                 existingMacEntry.get(), operTx);
 865                         } else {
 866                             elanForwardingEntriesHandler.addElanInterfaceForwardingTableList(elanInstanceName,
 867                                 interfaceName, staticMacEntry, operTx);
 868                         }
 869
 870                         if (isInterfaceOperational) {
 871                             // Setting SMAC, DMAC, UDMAC in this DPN and also in other
 872                             // DPNs
 873                             String macAddress = staticMacEntry.getMacAddress().getValue();
 874                             LOG.info(
 875                                 "programming smac and dmacs for {} on source and other DPNs for elan {} and interface"
 876                                     + " {}",
 877                                 macAddress, elanInstanceName, interfaceName);
 878                             EVENT_LOGGER.debug("ELAN-MacFlows, ADD {} Instance {} Mac {}",
 879                                     interfaceName, elanInstanceName, macAddress);
 880                             elanUtils.setupMacFlows(elanInstance, interfaceInfo, ElanConstants.STATIC_MAC_TIMEOUT,
 881                                 staticMacEntry.getMacAddress().getValue(), true, confTx);
 882                         }
 883                     }
 884
 885                     if (isInterfaceOperational) {
 886                         // Add MAC in TOR's remote MACs via OVSDB. Outside of the loop
 887                         // on purpose.
 888                         for (StaticMacEntries staticMacEntry : staticMacEntriesList) {
 889                             staticMacAddresses.add(staticMacEntry.getMacAddress());
 890                         }
 891                         elanL2GatewayUtils.scheduleAddDpnMacInExtDevices(elanInstance.getElanInstanceName(), dpId,
 892                             staticMacAddresses);
 893                     }
 894                 }
 895             }));
 896         }));
 897         futures.forEach(ElanUtils::waitForTransactionToComplete);
 898         if (isInterfaceOperational && !interfaceManager.isExternalInterface(interfaceName)) {
 899             //At this point, the interface is operational and D/SMAC flows have been configured, mark the port active
 900             try {
 901                 Port neutronPort = neutronVpnManager.getNeutronPort(interfaceName);
 902                 if (neutronPort != null) {
 903                     NeutronUtils.updatePortStatus(interfaceName, NeutronUtils.PORT_STATUS_ACTIVE, broker);
 904                 }
 905             } catch (IllegalArgumentException ex) {
 906                 LOG.trace("Interface: {} is not part of Neutron Network", interfaceName);
 907             }
 908         }
 909         return futures;
 910     }
 911
 912     protected void removeInterfaceStaticMacEntries(String elanInstanceName, String interfaceName,
 913             PhysAddress physAddress) {
 914         InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(interfaceName);
 915         InstanceIdentifier<MacEntry> macId = getMacEntryOperationalDataPath(elanInstanceName, physAddress);
 916         Optional<MacEntry> existingMacEntry = ElanUtils.read(broker,
 917                 LogicalDatastoreType.OPERATIONAL, macId);
 918
 919         if (!existingMacEntry.isPresent()) {
 920             return;
 921         }
 922
 923         MacEntry macEntry = new MacEntryBuilder().setMacAddress(physAddress).setInterface(interfaceName)
 924                 .withKey(new MacEntryKey(physAddress)).build();
 925         elanForwardingEntriesHandler.deleteElanInterfaceForwardingEntries(
 926                 elanInstanceCache.get(elanInstanceName).orNull(), interfaceInfo, macEntry);
 927     }
 928
 929     private boolean checkIfFirstInterface(String elanInterface, String elanInstanceName,
 930             Optional<DpnInterfaces> existingElanDpnInterfaces) {
 931         String routerPortUuid = ElanUtils.getRouterPordIdFromElanInstance(broker, elanInstanceName);
 932         if (!existingElanDpnInterfaces.isPresent()) {
 933             return true;
 934         }
 935         if (elanInterface.equals(elanInstanceName) || elanInterface.equals(routerPortUuid)) {
 936             return false;
 937         }
 938         DpnInterfaces dpnInterfaces = existingElanDpnInterfaces.get();
 939         int dummyInterfaceCount =  0;
 940         List<String> interfaces = dpnInterfaces.getInterfaces();
 941         if (interfaces == null) {
 942             return true;
 943         }
 944         if (interfaces.contains(routerPortUuid)) {
 945             dummyInterfaceCount++;
 946         }
 947         if (interfaces.contains(elanInstanceName)) {
 948             dummyInterfaceCount++;
 949         }
 950         return interfaces.size() == dummyInterfaceCount;
 951     }
 952
 953     private static InstanceIdentifier<MacEntry> getMacEntryOperationalDataPath(String elanName,
 954             PhysAddress physAddress) {
 955         return InstanceIdentifier.builder(ElanForwardingTables.class).child(MacTable.class, new MacTableKey(elanName))
 956                 .child(MacEntry.class, new MacEntryKey(physAddress)).build();
 957     }
 958
 959     private void installEntriesForElanInterface(ElanInstance elanInstance, ElanInterface elanInterface,
 960             InterfaceInfo interfaceInfo, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 961         if (!isOperational(interfaceInfo)) {
 962             LOG.warn("Interface {} is not operational", elanInterface.getName());
 963             EVENT_LOGGER.debug("ELAN-InterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
 964                     elanInterface.getName(), elanInstance.getElanInstanceName(), interfaceInfo.getOpState());
 965             return;
 966         }
 967         Uint64 dpId = interfaceInfo.getDpId();
 968         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
 969             elanUtils.setupTermDmacFlows(interfaceInfo, mdsalManager, confTx);
 970         }
 971         setupFilterEqualsTable(elanInstance, interfaceInfo, confTx);
 972         if (isFirstInterfaceInDpn) {
 973             // Terminating Service , UnknownDMAC Table.
 974             // The 1st ELAN Interface in a DPN must program the INTERNAL_TUNNEL_TABLE, but only if the network type
 975             // for ELAN Instance is VxLAN
 976             if (isVxlanNetworkOrVxlanSegment(elanInstance)) {
 977                 setupTerminateServiceTable(elanInstance, dpId, confTx);
 978             }
 979             setupUnknownDMacTable(elanInstance, dpId, confTx);
 980             /*
 981              * Install remote DMAC flow. This is required since this DPN is
 982              * added later to the elan instance and remote DMACs of other
 983              * interfaces in this elan instance are not present in the current
 984              * dpn.
 985              */
 986             if (!interfaceManager.isExternalInterface(interfaceInfo.getInterfaceName())) {
 987                 LOG.info("Programming remote dmac flows on the newly connected dpn {} for elan {} ", dpId,
 988                         elanInstance.getElanInstanceName());
 989                 programRemoteDmacFlow(elanInstance, interfaceInfo, confTx);
 990             }
 991         }
 992         // bind the Elan service to the Interface
 993         bindService(elanInstance, elanInterface, interfaceInfo.getInterfaceTag(), confTx);
 994     }
 995
 996     private void installEntriesForFirstInterfaceonDpn(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
 997             DpnInterfaces dpnInterfaces, boolean isFirstInterfaceInDpn, TypedWriteTransaction<Configuration> confTx) {
 998         if (!isOperational(interfaceInfo)) {
 999             LOG.warn("Entries for interface on dpn {} not installed since interface {} is not operational",
1000                     dpnInterfaces.getDpId(), interfaceInfo.getInterfaceName());
1001             EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {} Interface Status {}, returning",
1002                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1003             return;
1004         }
1005         EVENT_LOGGER.debug("ELAN-1stInterfaceEntries, ADD {} Instance {}",
1006                 interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName());
1007         // LocalBroadcast Group creation with elan-Interfaces
1008         LOG.info("Installing entries for interface {} on dpn {}", interfaceInfo.getInterfaceName(),
1009                 dpnInterfaces.getDpId());
1010         setupLocalBroadcastGroups(elanInfo, dpnInterfaces, interfaceInfo, confTx);
1011         if (isFirstInterfaceInDpn) {
1012             LOG.trace("waitTimeForSyncInstall is {}", WAIT_TIME_FOR_SYNC_INSTALL);
1013             try {
1014                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1015             } catch (InterruptedException e1) {
1016                 LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInfo);
1017             }
1018         }
1019     }
1020
1021     public InstanceIdentifier<Group> getGroupIid(ElanInstance elanInfo, Uint64 dpnId) {
1022         long remoteBcGroupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1023         return InstanceIdentifier.builder(Nodes.class)
1024                 .child(Node.class, new NodeKey(new org.opendaylight.yang.gen.v1.urn.opendaylight
1025                         .inventory.rev130819.NodeId("openflow:" + dpnId.toString())))
1026                 .augmentation(FlowCapableNode.class)
1027                 .child(Group.class, new GroupKey(new GroupId(remoteBcGroupId))).build();
1028     }
1029
1030     public void scheduleElanInterfaceWorkerAfterRemoteBcGroup(ElanInstance elanInfo,
1031                                                               InterfaceInfo interfaceInfo,
1032                                                               DpnInterfaces dpnInterfaces,
1033                                                               boolean isFirstInterfaceInDpn,
1034                                                               ElanInterface elanInterface) {
1035         if (!isOperational(interfaceInfo)) {
1036             LOG.debug("Interface {} is not operational", elanInterface.getName());
1037             return;
1038         }
1039         String elanInterfaceJobKey = ElanUtils.getElanInterfaceJobKey(interfaceInfo.getInterfaceName());
1040         InterfaceAddWorkerOnElanInterface addWorker = new InterfaceAddWorkerOnElanInterface(elanInterfaceJobKey,
1041                 elanInterface, interfaceInfo, elanInfo, isFirstInterfaceInDpn, this);
1042         InstanceIdentifier<Group> groupInstanceId = getGroupIid(elanInfo, dpnInterfaces.getDpId());
1043         elanGroupCache.addJobToWaitList(groupInstanceId, () -> {
1044             jobCoordinator.enqueueJob(elanInterfaceJobKey, addWorker, ElanConstants.JOB_MAX_RETRIES);
1045         });
1046     }
1047
1048     public void setupFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1049             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1050         int ifTag = interfaceInfo.getInterfaceTag();
1051         Flow flow = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1052                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"), 11, elanInfo.getElanInstanceName(),
1053                 0, 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1054                 ElanUtils.getTunnelIdMatchForFilterEqualsLPortTag(ifTag),
1055                 elanUtils.getInstructionsInPortForOutGroup(interfaceInfo.getInterfaceName()));
1056
1057         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flow);
1058         LOG.trace("Filter equals table(55) flow entry created on dpn: {} for interface port: {}",
1059                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1060
1061         Flow flowEntry = MDSALUtil.buildFlowNew(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1062                 getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"), 12, elanInfo.getElanInstanceName(), 0,
1063                 0, Uint64.valueOf(ElanConstants.COOKIE_ELAN_FILTER_EQUALS.toJava().add(BigInteger.valueOf(ifTag))),
1064                 getMatchesForFilterEqualsLPortTag(ifTag), MDSALUtil.buildInstructionsDrop());
1065
1066         mdsalManager.addFlow(writeFlowGroupTx, interfaceInfo.getDpId(), flowEntry);
1067         LOG.trace("Filter equals table(55) drop flow entry created on dpn: {} for interface port: {}",
1068                 interfaceInfo.getDpId(), interfaceInfo.getPortName());
1069     }
1070
1071     public void removeFilterEqualsTable(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1072             TypedReadWriteTransaction<Configuration> flowTx) throws ExecutionException, InterruptedException {
1073         int ifTag = interfaceInfo.getInterfaceTag();
1074         Flow flow = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1075             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "group"));
1076
1077         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flow);
1078
1079         Flow flowEntity = MDSALUtil.buildFlow(NwConstants.ELAN_FILTER_EQUALS_TABLE,
1080             getFlowRef(NwConstants.ELAN_FILTER_EQUALS_TABLE, ifTag, "drop"));
1081
1082         mdsalManager.removeFlow(flowTx, interfaceInfo.getDpId(), flowEntity);
1083     }
1084
1085     private static List<MatchInfo> buildMatchesForVni(Uint64 vni) {
1086         List<MatchInfo> mkMatches = new ArrayList<>();
1087         MatchInfo match = new MatchTunnelId(vni);
1088         mkMatches.add(match);
1089         return mkMatches;
1090     }
1091
1092     private static List<InstructionInfo> getInstructionsForOutGroup(long groupId) {
1093         List<InstructionInfo> mkInstructions = new ArrayList<>();
1094         mkInstructions.add(new InstructionWriteActions(Collections.singletonList(new ActionGroup(groupId))));
1095         return mkInstructions;
1096     }
1097
1098     private static List<MatchInfo> getMatchesForElanTag(long elanTag, boolean isSHFlagSet) {
1099         List<MatchInfo> mkMatches = new ArrayList<>();
1100         // Matching metadata
1101         mkMatches.add(new MatchMetadata(
1102                 ElanUtils.getElanMetadataLabel(elanTag, isSHFlagSet), MetaDataUtil.METADATA_MASK_SERVICE_SH_FLAG));
1103         return mkMatches;
1104     }
1105
1106     /**
1107      * Builds the list of instructions to be installed in the INTERNAL_TUNNEL_TABLE (36) / EXTERNAL_TUNNEL_TABLE (38)
1108      * which so far consists of writing the elanTag in metadata and send the packet to ELAN_DMAC_TABLE.
1109      *
1110      * @param elanTag
1111      *            elanTag to be written in metadata when flow is selected
1112      * @return the instructions ready to be installed in a flow
1113      */
1114     private static List<InstructionInfo> getInstructionsIntOrExtTunnelTable(Uint32 elanTag) {
1115         List<InstructionInfo> mkInstructions = new ArrayList<>();
1116         mkInstructions.add(new InstructionWriteMetadata(ElanHelper.getElanMetadataLabel(elanTag.longValue()),
1117                 ElanHelper.getElanMetadataMask()));
1118         /* applicable for EXTERNAL_TUNNEL_TABLE only
1119         * TODO: We should point to SMAC or DMAC depending on a configuration property to enable mac learning
1120         */
1121         mkInstructions.add(new InstructionGotoTable(NwConstants.ELAN_DMAC_TABLE));
1122         return mkInstructions;
1123     }
1124
1125     // Install DMAC entry on dst DPN
1126     public List<ListenableFuture<Void>> installDMacAddressTables(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1127             Uint64 dstDpId) {
1128         String interfaceName = interfaceInfo.getInterfaceName();
1129         ElanInterfaceMac elanInterfaceMac = elanUtils.getElanInterfaceMacByInterfaceName(interfaceName);
1130         if (elanInterfaceMac != null && elanInterfaceMac.getMacEntry() != null) {
1131             List<MacEntry> macEntries = elanInterfaceMac.getMacEntry();
1132             return Collections.singletonList(ElanUtils.waitForTransactionToComplete(
1133                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
1134                     for (MacEntry macEntry : macEntries) {
1135                         String macAddress = macEntry.getMacAddress().getValue();
1136                         LOG.info("Installing remote dmac for mac address {} and interface {}", macAddress,
1137                             interfaceName);
1138                         try (Acquired lock = ElanUtils.lockElanMacDPN(elanInfo.getElanTag().toJava(), macAddress,
1139                             interfaceInfo.getDpId())) {
1140                             LOG.info("Acquired lock for mac : {}, proceeding with remote dmac install operation",
1141                                 macAddress);
1142                             elanUtils.setupDMacFlowOnRemoteDpn(elanInfo, interfaceInfo, dstDpId, macAddress, tx);
1143                         }
1144                     }
1145                 })));
1146         }
1147         return emptyList();
1148     }
1149
1150     private static void createDropBucket(List<Bucket> listBucket) {
1151         List<Action> actionsInfos = new ArrayList<>();
1152         actionsInfos.add(new ActionDrop().buildAction());
1153         Bucket dropBucket = MDSALUtil.buildBucket(actionsInfos, MDSALUtil.GROUP_WEIGHT, 0, MDSALUtil.WATCH_PORT,
1154                 MDSALUtil.WATCH_GROUP);
1155         listBucket.add(dropBucket);
1156     }
1157
1158     private void setupLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1159             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1160         if (!isOperational(interfaceInfo)) {
1161             EVENT_LOGGER.debug("ELAN-LBG, ADD {} Instance {} Interface Status {}, returning",
1162                     interfaceInfo.getInterfaceName(), elanInfo.getElanInstanceName(), interfaceInfo.getOpState());
1163             return;
1164         }
1165         setupStandardLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1166         setupLeavesLocalBroadcastGroups(elanInfo, newDpnInterface, interfaceInfo, confTx);
1167     }
1168
1169     private void setupStandardLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1170             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1171         List<Bucket> listBucket = new ArrayList<>();
1172         int bucketId = 0;
1173         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1174
1175         List<String> interfaces = new ArrayList<>();
1176         if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1177             interfaces = newDpnInterface.getInterfaces();
1178         }
1179         for (String ifName : interfaces) {
1180             // In case if there is a InterfacePort in the cache which is not in
1181             // operational state, skip processing it
1182             InterfaceInfo ifInfo = interfaceManager
1183                     .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1184             if (!isOperational(ifInfo)) {
1185                 continue;
1186             }
1187
1188             if (!interfaceManager.isExternalInterface(ifName)) {
1189                 listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1190                         MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1191                 bucketId++;
1192             }
1193         }
1194
1195         Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1196                 MDSALUtil.buildBucketLists(listBucket));
1197         LOG.trace("installing the localBroadCast Group:{}", group);
1198         mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1199     }
1200
1201     private void setupLeavesLocalBroadcastGroups(ElanInstance elanInfo, DpnInterfaces newDpnInterface,
1202             InterfaceInfo interfaceInfo, TypedWriteTransaction<Configuration> confTx) {
1203         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1204         if (etreeInstance != null) {
1205             List<Bucket> listBucket = new ArrayList<>();
1206             int bucketId = 0;
1207
1208             List<String> interfaces = new ArrayList<>();
1209             if (newDpnInterface != null && newDpnInterface.getInterfaces() != null) {
1210                 interfaces = newDpnInterface.getInterfaces();
1211             }
1212             for (String ifName : interfaces) {
1213                 // In case if there is a InterfacePort in the cache which is not
1214                 // in
1215                 // operational state, skip processing it
1216                 InterfaceInfo ifInfo = interfaceManager
1217                         .getInterfaceInfoFromOperationalDataStore(ifName, interfaceInfo.getInterfaceType());
1218                 if (!isOperational(ifInfo)) {
1219                     continue;
1220                 }
1221
1222                 if (!interfaceManager.isExternalInterface(ifName)) {
1223                     // only add root interfaces
1224                     bucketId = addInterfaceIfRootInterface(bucketId, ifName, listBucket, ifInfo);
1225                 }
1226             }
1227
1228             if (listBucket.isEmpty()) { // No Buckets
1229                 createDropBucket(listBucket);
1230             }
1231
1232             long etreeLeafTag = etreeInstance.getEtreeLeafTagVal().getValue().toJava();
1233             long groupId = ElanUtils.getEtreeLeafLocalBCGId(etreeLeafTag);
1234             Group group = MDSALUtil.buildGroup(groupId, elanInfo.getElanInstanceName(), GroupTypes.GroupAll,
1235                     MDSALUtil.buildBucketLists(listBucket));
1236             LOG.trace("installing the localBroadCast Group:{}", group);
1237             mdsalManager.addGroup(confTx, interfaceInfo.getDpId(), group);
1238         }
1239     }
1240
1241     private int addInterfaceIfRootInterface(int bucketId, String ifName, List<Bucket> listBucket,
1242             InterfaceInfo ifInfo) {
1243         Optional<EtreeInterface> etreeInterface = elanInterfaceCache.getEtreeInterface(ifName);
1244         if (etreeInterface.isPresent() && etreeInterface.get().getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1245             listBucket.add(MDSALUtil.buildBucket(getInterfacePortActions(ifInfo), MDSALUtil.GROUP_WEIGHT, bucketId,
1246                     MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP));
1247             bucketId++;
1248         }
1249         return bucketId;
1250     }
1251
1252     public void removeLocalBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1253             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1254             throws ExecutionException, InterruptedException {
1255         Uint64 dpnId = interfaceInfo.getDpId();
1256         long groupId = ElanUtils.getElanLocalBCGId(elanInfo.getElanTag().toJava());
1257         LOG.trace("deleted the localBroadCast Group:{}", groupId);
1258         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1259     }
1260
1261     public void removeElanBroadcastGroup(ElanInstance elanInfo, InterfaceInfo interfaceInfo,
1262             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx)
1263             throws ExecutionException, InterruptedException {
1264         Uint64 dpnId = interfaceInfo.getDpId();
1265         long groupId = ElanUtils.getElanRemoteBCGId(elanInfo.getElanTag().toJava());
1266         LOG.trace("deleting the remoteBroadCast group:{}", groupId);
1267         mdsalManager.removeGroup(deleteFlowGroupTx, dpnId, groupId);
1268     }
1269
1270     /**
1271      * Installs a flow in the External Tunnel table consisting in translating
1272      * the VNI retrieved from the packet that came over a tunnel with a TOR into
1273      * elanTag that will be used later in the ELANs pipeline.
1274      * @param dpnId the dpn id
1275      */
1276     private void setExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1277             TypedWriteTransaction<Configuration> confTx) {
1278         Uint32 elanTag = elanInfo.getElanTag();
1279         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.EXTERNAL_TUNNEL_TABLE,
1280                 getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanTag.longValue()), 5, // prio
1281                 elanInfo.getElanInstanceName(), // flowName
1282                 0, // idleTimeout
1283                 0, // hardTimeout
1284                 Uint64.valueOf(ITMConstants.COOKIE_ITM_EXTERNAL.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1285                 buildMatchesForVni(Uint64.valueOf(ElanUtils.getVxlanSegmentationId(elanInfo).longValue())),
1286                 getInstructionsIntOrExtTunnelTable(elanTag));
1287
1288         mdsalManager.addFlow(confTx, flowEntity);
1289     }
1290
1291     /**
1292      * Removes, from External Tunnel table, the flow that translates from VNI to
1293      * elanTag. Important: ensure this method is only called whenever there is
1294      * no other ElanInterface in the specified DPN
1295      *
1296      * @param dpnId DPN whose Ext Tunnel table is going to be modified
1297      */
1298     private void unsetExternalTunnelTable(Uint64 dpnId, ElanInstance elanInfo,
1299             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
1300         // TODO: Use DataStoreJobCoordinator in order to avoid that removing the
1301         // last ElanInstance plus
1302         // adding a new one does (almost at the same time) are executed in that
1303         // exact order
1304
1305         String flowId = getFlowRef(NwConstants.EXTERNAL_TUNNEL_TABLE, elanInfo.getElanTag().toJava());
1306         FlowEntity flowEntity = new FlowEntityBuilder()
1307             .setDpnId(dpnId)
1308             .setTableId(NwConstants.EXTERNAL_TUNNEL_TABLE)
1309             .setFlowId(flowId)
1310             .build();
1311         mdsalManager.removeFlow(confTx, flowEntity);
1312     }
1313
1314     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1315             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1316         setupTerminateServiceTable(elanInfo, dpId, elanInfo.getElanTag(), writeFlowGroupTx);
1317         setupEtreeTerminateServiceTable(elanInfo, dpId, writeFlowGroupTx);
1318     }
1319
1320     public void setupTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId, Uint32 elanTag,
1321             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1322         List<? extends MatchInfoBase> listMatchInfoBase;
1323         List<InstructionInfo> instructionInfos;
1324         Uint32 serviceId;
1325         if (!elanUtils.isOpenstackVniSemanticsEnforced()) {
1326             serviceId = elanTag;
1327             listMatchInfoBase = ElanUtils.getTunnelMatchesForServiceId(elanTag);
1328             instructionInfos = getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag.longValue()));
1329         } else {
1330             serviceId = ElanUtils.getVxlanSegmentationId(elanInfo);
1331             listMatchInfoBase = buildMatchesForVni(Uint64.valueOf(serviceId));
1332             instructionInfos = getInstructionsIntOrExtTunnelTable(elanTag);
1333         }
1334         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE,
1335                 getFlowRef(NwConstants.INTERNAL_TUNNEL_TABLE, serviceId.longValue()), 5,
1336                 String.format("%s:%s", "ITM Flow Entry ", elanTag.toString()), 0, 0,
1337                 Uint64.valueOf(ITMConstants.COOKIE_ITM.toJava().add(BigInteger.valueOf(elanTag.longValue()))),
1338                 listMatchInfoBase, instructionInfos);
1339         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1340         LOG.info("Installed internal tunnel table (36) flow entry on dpn: {} for elan: {}, tag: {}", dpId,
1341                 elanInfo.getElanInstanceName(), elanTag);
1342     }
1343
1344     private void setupEtreeTerminateServiceTable(ElanInstance elanInfo, Uint64 dpId,
1345             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1346         EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1347         if (etreeInstance != null) {
1348             setupTerminateServiceTable(elanInfo, dpId,
1349                 etreeInstance.getEtreeLeafTagVal().getValue(), writeFlowGroupTx);
1350         }
1351     }
1352
1353     public void setupUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId,
1354             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1355         long elanTag = elanInfo.getElanTag().toJava();
1356         installLocalUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1357         installRemoteUnknownFlow(elanInfo, dpId, elanTag, writeFlowGroupTx);
1358         setupEtreeUnknownDMacTable(elanInfo, dpId, elanTag, writeFlowGroupTx);
1359     }
1360
1361     private void setupEtreeUnknownDMacTable(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1362             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1363         EtreeLeafTagName etreeLeafTag = elanEtreeUtils.getEtreeLeafTagByElanTag(elanTag);
1364         if (etreeLeafTag != null) {
1365             long leafTag = etreeLeafTag.getEtreeLeafTag().getValue().toJava();
1366             installRemoteUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1367             installLocalUnknownFlow(elanInfo, dpId, leafTag, writeFlowGroupTx);
1368         }
1369     }
1370
1371     private void installLocalUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1372             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1373         FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1374                 getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */false),
1375                 5, elanInfo.getElanInstanceName(), 0, 0,
1376                 Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1377                 getMatchesForElanTag(elanTag, /* SH flag */false),
1378                 getInstructionsForOutGroup(ElanUtils.getElanRemoteBCGId(elanTag)));
1379
1380         mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1381         LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan: {}, tag: {}",
1382                 dpId, elanInfo.getElanInstanceName(), elanTag);
1383     }
1384
1385     private void installRemoteUnknownFlow(ElanInstance elanInfo, Uint64 dpId, long elanTag,
1386             TypedWriteTransaction<Configuration> writeFlowGroupTx) {
1387         // only if ELAN can connect to external network, perform the following
1388
1389         if (isVxlanNetworkOrVxlanSegment(elanInfo) || ElanUtils.isVlan(elanInfo) || ElanUtils.isFlat(elanInfo)) {
1390             FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1391                     getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE, elanTag,/* SH flag */true),
1392                     5, elanInfo.getElanInstanceName(), 0, 0,
1393                     Uint64.valueOf(ElanConstants.COOKIE_ELAN_UNKNOWN_DMAC.toJava().add(BigInteger.valueOf(elanTag))),
1394                     getMatchesForElanTag(elanTag, /* SH flag */true),
1395                     getInstructionsForOutGroup(ElanUtils.getElanLocalBCGId(elanTag)));
1396             mdsalManager.addFlow(writeFlowGroupTx, flowEntity);
1397             LOG.trace("Installed unknown dmac table (53) flow entry on dpn: {} for elan connected to "
1398                     + "external network: {}, tag: {}", dpId, elanInfo.getElanInstanceName(), elanTag);
1399         }
1400     }
1401
1402
1403     private void removeUnknownDmacFlow(Uint64 dpId, ElanInstance elanInfo,
1404             TypedReadWriteTransaction<Configuration> deleteFlowGroupTx, long elanTag)
1405             throws ExecutionException, InterruptedException {
1406         Flow flow = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1407                 elanTag, SH_FLAG_UNSET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE).build();
1408         mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow);
1409
1410         if (isVxlanNetworkOrVxlanSegment(elanInfo)) {
1411             Flow flow2 = new FlowBuilder().setId(new FlowId(getUnknownDmacFlowRef(NwConstants.ELAN_UNKNOWN_DMAC_TABLE,
1412                     elanTag, SH_FLAG_SET))).setTableId(NwConstants.ELAN_UNKNOWN_DMAC_TABLE)
1413                     .build();
1414             mdsalManager.removeFlow(deleteFlowGroupTx, dpId, flow2);
1415         }
1416     }
1417
1418     private void removeDefaultTermFlow(Uint64 dpId, long elanTag) {
1419         elanUtils.removeTerminatingServiceAction(dpId, (int) elanTag);
1420     }
1421
1422     private void bindService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1423             TypedWriteTransaction<Configuration> tx) {
1424         if (isStandardElanService(elanInterface)) {
1425             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(),
1426                     elanInterface.getName(), lportTag, tx);
1427         } else { // Etree service
1428             bindEtreeService(elanInfo, elanInterface, lportTag, tx);
1429         }
1430     }
1431
1432     private void bindElanService(long elanTag, String elanInstanceName, String interfaceName, int lportTag,
1433             TypedWriteTransaction<Configuration> tx) {
1434         int instructionKey = 0;
1435         List<Instruction> instructions = new ArrayList<>();
1436         instructions.add(MDSALUtil.buildAndGetWriteMetadaInstruction(ElanHelper.getElanMetadataLabel(elanTag),
1437                 MetaDataUtil.METADATA_MASK_SERVICE, ++instructionKey));
1438
1439         List<Action> actions = new ArrayList<>();
1440         actions.add(new ActionRegLoad(0, NxmNxReg1.class, 0, ElanConstants.INTERFACE_TAG_LENGTH - 1,
1441                 lportTag).buildAction());
1442         actions.add(new ActionRegLoad(1, ElanConstants.ELAN_REG_ID, 0, ElanConstants.ELAN_TAG_LENGTH - 1,
1443                 elanTag).buildAction());
1444         instructions.add(MDSALUtil.buildApplyActionsInstruction(actions, ++instructionKey));
1445
1446         instructions.add(MDSALUtil.buildAndGetGotoTableInstruction(NwConstants.ARP_CHECK_TABLE,
1447                 ++instructionKey));
1448
1449         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1450         BoundServices serviceInfo = ElanUtils.getBoundServices(
1451                 String.format("%s.%s.%s", "elan", elanInstanceName, interfaceName), elanServiceIndex,
1452                 NwConstants.ELAN_SERVICE_INDEX, NwConstants.COOKIE_ELAN_INGRESS_TABLE, instructions);
1453         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1454         Optional<BoundServices> existingElanService = ElanUtils.read(broker, LogicalDatastoreType.CONFIGURATION,
1455                 bindServiceId);
1456         if (!existingElanService.isPresent()) {
1457             tx.put(bindServiceId, serviceInfo, CREATE_MISSING_PARENTS);
1458             LOG.trace("Done binding elan service for elan: {} for interface: {}", elanInstanceName, interfaceName);
1459         }
1460     }
1461
1462     private void bindEtreeService(ElanInstance elanInfo, ElanInterface elanInterface, int lportTag,
1463             TypedWriteTransaction<Configuration> tx) {
1464         if (elanInterface.augmentation(EtreeInterface.class).getEtreeInterfaceType() == EtreeInterfaceType.Root) {
1465             bindElanService(elanInfo.getElanTag().toJava(), elanInfo.getElanInstanceName(), elanInterface.getName(),
1466                     lportTag, tx);
1467         } else {
1468             EtreeInstance etreeInstance = elanInfo.augmentation(EtreeInstance.class);
1469             if (etreeInstance == null) {
1470                 LOG.error("EtreeInterface {} is associated with a non EtreeInstance: {}",
1471                         elanInterface.getName(), elanInfo.getElanInstanceName());
1472             } else {
1473                 bindElanService(etreeInstance.getEtreeLeafTagVal().getValue().toJava(), elanInfo.getElanInstanceName(),
1474                         elanInterface.getName(), lportTag, tx);
1475             }
1476         }
1477     }
1478
1479     private static boolean isStandardElanService(ElanInterface elanInterface) {
1480         return elanInterface.augmentation(EtreeInterface.class) == null;
1481     }
1482
1483     protected void unbindService(String interfaceName, TypedReadWriteTransaction<Configuration> tx)
1484             throws ExecutionException, InterruptedException {
1485         short elanServiceIndex = ServiceIndex.getIndex(NwConstants.ELAN_SERVICE_NAME, NwConstants.ELAN_SERVICE_INDEX);
1486         InstanceIdentifier<BoundServices> bindServiceId = ElanUtils.buildServiceId(interfaceName, elanServiceIndex);
1487         if (tx.read(bindServiceId).get().isPresent()) {
1488             tx.delete(bindServiceId);
1489         }
1490     }
1491
1492     private static String getFlowRef(long tableId, long elanTag) {
1493         return String.valueOf(tableId) + elanTag;
1494     }
1495
1496     private static String getFlowRef(long tableId, long elanTag, String flowName) {
1497         return new StringBuilder().append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(elanTag)
1498                 .append(NwConstants.FLOWID_SEPARATOR).append(flowName).toString();
1499     }
1500
1501     private static String getUnknownDmacFlowRef(long tableId, long elanTag, boolean shFlag) {
1502         return String.valueOf(tableId) + elanTag + shFlag;
1503     }
1504
1505     private static List<Action> getInterfacePortActions(InterfaceInfo interfaceInfo) {
1506         List<Action> listAction = new ArrayList<>();
1507         int actionKey = 0;
1508         listAction.add(
1509             new ActionSetFieldTunnelId(Uint64.valueOf(interfaceInfo.getInterfaceTag())).buildAction(actionKey));
1510         actionKey++;
1511         listAction.add(new ActionNxResubmit(NwConstants.ELAN_FILTER_EQUALS_TABLE).buildAction(actionKey));
1512         return listAction;
1513     }
1514
1515     private static DpnInterfaces updateElanDpnInterfacesList(String elanInstanceName, Uint64 dpId,
1516             List<String> interfaceNames, TypedWriteTransaction<Operational> tx) {
1517         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1518                 .withKey(new DpnInterfacesKey(dpId)).build();
1519         tx.put(ElanUtils.getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId), dpnInterface,
1520                 CREATE_MISSING_PARENTS);
1521         LOG.trace("Updated operational dpn interfaces for elan: {} with interfaces: {}", elanInstanceName,
1522                 interfaceNames);
1523         return dpnInterface;
1524     }
1525
1526     /**
1527      * Delete elan dpn interface from operational DS.
1528      *
1529      * @param elanInstanceName
1530      *            the elan instance name
1531      * @param dpId
1532      *            the dp id
1533      */
1534     private static void deleteElanDpnInterface(String elanInstanceName, Uint64 dpId,
1535             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1536         InstanceIdentifier<DpnInterfaces> dpnInterfacesId = ElanUtils
1537                 .getElanDpnInterfaceOperationalDataPath(elanInstanceName, dpId);
1538         Optional<DpnInterfaces> dpnInterfaces = tx.read(dpnInterfacesId).get();
1539         if (dpnInterfaces.isPresent()) {
1540             tx.delete(dpnInterfacesId);
1541         }
1542     }
1543
1544     private static DpnInterfaces createElanInterfacesList(String elanInstanceName, String interfaceName,
1545             Uint64 dpId, TypedWriteTransaction<Operational> tx) {
1546         List<String> interfaceNames = new ArrayList<>();
1547         interfaceNames.add(interfaceName);
1548         DpnInterfaces dpnInterface = new DpnInterfacesBuilder().setDpId(dpId).setInterfaces(interfaceNames)
1549                 .withKey(new DpnInterfacesKey(dpId)).build();
1550         return dpnInterface;
1551     }
1552
1553     private static void createElanInterfaceTablesList(String interfaceName, TypedReadWriteTransaction<Operational> tx)
1554             throws ExecutionException, InterruptedException {
1555         InstanceIdentifier<ElanInterfaceMac> elanInterfaceMacTables = ElanUtils
1556                 .getElanInterfaceMacEntriesOperationalDataPath(interfaceName);
1557         Optional<ElanInterfaceMac> interfaceMacTables = tx.read(elanInterfaceMacTables).get();
1558         // Adding new Elan Interface Port to the operational DataStore without
1559         // Static-Mac Entries..
1560         if (!interfaceMacTables.isPresent()) {
1561             ElanInterfaceMac elanInterfaceMacTable = new ElanInterfaceMacBuilder().setElanInterface(interfaceName)
1562                     .withKey(new ElanInterfaceMacKey(interfaceName)).build();
1563             tx.put(ElanUtils.getElanInterfaceMacEntriesOperationalDataPath(interfaceName), elanInterfaceMacTable,
1564                     CREATE_MISSING_PARENTS);
1565             LOG.trace("Created interface MAC table for interface: {}", interfaceName);
1566         }
1567     }
1568
1569     private static void createElanStateList(String elanInstanceName, String interfaceName,
1570             TypedReadWriteTransaction<Operational> tx) throws ExecutionException, InterruptedException {
1571         InstanceIdentifier<Elan> elanInstance = ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName);
1572         Optional<Elan> elanInterfaceLists = tx.read(elanInstance).get();
1573         // Adding new Elan Interface Port to the operational DataStore without
1574         // Static-Mac Entries..
1575         if (elanInterfaceLists.isPresent()) {
1576             List<String> interfaceLists = new ArrayList<>();
1577             interfaceLists.add(interfaceName);
1578             List<String> existingInterfaceLists = elanInterfaceLists.get().getElanInterfaces();
1579             if (existingInterfaceLists != null && !existingInterfaceLists.isEmpty()) {
1580                 existingInterfaceLists.forEach(iface -> interfaceLists.add(iface));
1581             }
1582
1583             Elan elanState = new ElanBuilder().setName(elanInstanceName).setElanInterfaces(interfaceLists)
1584                     .withKey(new ElanKey(elanInstanceName)).build();
1585             tx.put(ElanUtils.getElanInstanceOperationalDataPath(elanInstanceName), elanState, CREATE_MISSING_PARENTS);
1586             LOG.trace("Updated operational elan state for elan: {} with interfaces: {}", elanInstanceName,
1587                     interfaceLists);
1588         }
1589     }
1590
1591     private static boolean isOperational(InterfaceInfo interfaceInfo) {
1592         return interfaceInfo != null && interfaceInfo.getAdminState() == InterfaceInfo.InterfaceAdminState.ENABLED;
1593     }
1594
1595     @SuppressWarnings("checkstyle:IllegalCatch")
1596     public void handleInternalTunnelStateEvent(Uint64 srcDpId, Uint64 dstDpId) {
1597         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1598         LOG.trace("processing tunnel state event for srcDpId {} dstDpId {}"
1599                 + " and dpnInterfaceList {}", srcDpId, dstDpId, dpnInterfaceLists);
1600         if (dpnInterfaceLists == null) {
1601             return;
1602         }
1603         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1604         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1605             int cnt = 0;
1606             String elanName = elanDpns.getElanInstanceName();
1607             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1608             if (elanInfo == null) {
1609                 LOG.warn("ELAN Info is null for elanName {} that does exist in elanDpnInterfaceList, "
1610                         + "skipping this ELAN for tunnel handling", elanName);
1611                 continue;
1612             }
1613             if (!isVxlanNetworkOrVxlanSegment(elanInfo)) {
1614                 LOG.debug("Ignoring internal tunnel state event for Flat/Vlan elan {}", elanName);
1615                 continue;
1616             }
1617             List<DpnInterfaces> dpnInterfaces = elanDpns.getDpnInterfaces();
1618             if (dpnInterfaces == null) {
1619                 continue;
1620             }
1621             DpnInterfaces dstDpnIf = null;
1622             for (DpnInterfaces dpnIf : dpnInterfaces) {
1623                 Uint64 dpnIfDpId = dpnIf.getDpId();
1624                 if (Objects.equals(dpnIfDpId, srcDpId)) {
1625                     cnt++;
1626                 } else if (Objects.equals(dpnIfDpId, dstDpId)) {
1627                     cnt++;
1628                     dstDpnIf = dpnIf;
1629                 }
1630             }
1631             if (cnt == 2) {
1632                 LOG.info("Elan instance:{} is present b/w srcDpn:{} and dstDpn:{}", elanName, srcDpId, dstDpId);
1633                 final DpnInterfaces finalDstDpnIf = dstDpnIf; // var needs to be final so it can be accessed in lambda
1634                 jobCoordinator.enqueueJob(elanName, () -> {
1635                     // update Remote BC Group
1636                     LOG.trace("procesing elan remote bc group for tunnel event {}", elanInfo);
1637                     try {
1638                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1639                             confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, srcDpId,
1640                                 confTx)).get();
1641                     } catch (RuntimeException e) {
1642                         LOG.error("Error while adding remote bc group for {} on dpId {} ", elanName, srcDpId);
1643                     }
1644                     Set<String> interfaceLists = new HashSet<>();
1645                     interfaceLists.addAll(finalDstDpnIf.getInterfaces());
1646                     for (String ifName : interfaceLists) {
1647                         jobCoordinator.enqueueJob(ElanUtils.getElanInterfaceJobKey(ifName), () -> {
1648                             LOG.info("Processing tunnel up event for elan {} and interface {}", elanName, ifName);
1649                             InterfaceInfo interfaceInfo = interfaceManager.getInterfaceInfo(ifName);
1650                             if (isOperational(interfaceInfo)) {
1651                                 return installDMacAddressTables(elanInfo, interfaceInfo, srcDpId);
1652                             }
1653                             return emptyList();
1654                         }, ElanConstants.JOB_MAX_RETRIES);
1655                     }
1656                     return emptyList();
1657                 }, ElanConstants.JOB_MAX_RETRIES);
1658             }
1659
1660         }
1661     }
1662
1663     /**
1664      * Handle external tunnel state event.
1665      *
1666      * @param externalTunnel
1667      *            the external tunnel
1668      * @param intrf
1669      *            the interface
1670      */
1671     void handleExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1672         if (!validateExternalTunnelStateEvent(externalTunnel, intrf)) {
1673             return;
1674         }
1675         // dpId/externalNodeId will be available either in source or destination
1676         // based on the tunnel end point
1677         Uint64 dpId = null;
1678         NodeId externalNodeId = null;
1679         if (StringUtils.isNumeric(externalTunnel.getSourceDevice())) {
1680             dpId = Uint64.valueOf(externalTunnel.getSourceDevice());
1681             externalNodeId = new NodeId(externalTunnel.getDestinationDevice());
1682         } else if (StringUtils.isNumeric(externalTunnel.getDestinationDevice())) {
1683             dpId = Uint64.valueOf(externalTunnel.getDestinationDevice());
1684             externalNodeId = new NodeId(externalTunnel.getSourceDevice());
1685         }
1686         if (dpId == null || externalNodeId == null) {
1687             LOG.error("Dp ID / externalNodeId not found in external tunnel {}", externalTunnel);
1688             return;
1689         }
1690
1691         ElanDpnInterfaces dpnInterfaceLists = elanUtils.getElanDpnInterfacesList();
1692         if (dpnInterfaceLists == null) {
1693             return;
1694         }
1695         List<ElanDpnInterfacesList> elanDpnIf = dpnInterfaceLists.nonnullElanDpnInterfacesList();
1696         for (ElanDpnInterfacesList elanDpns : elanDpnIf) {
1697             String elanName = elanDpns.getElanInstanceName();
1698             ElanInstance elanInfo = elanInstanceCache.get(elanName).orNull();
1699
1700             DpnInterfaces dpnInterfaces = elanUtils.getElanInterfaceInfoByElanDpn(elanName, dpId);
1701             if (elanInfo == null || dpnInterfaces == null || dpnInterfaces.getInterfaces() == null
1702                     || dpnInterfaces.getInterfaces().isEmpty()) {
1703                 continue;
1704             }
1705             LOG.debug("Elan instance:{} is present in Dpn:{} ", elanName, dpId);
1706
1707             final Uint64 finalDpId = dpId;
1708             LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1709                 confTx -> elanL2GatewayMulticastUtils.setupElanBroadcastGroups(elanInfo, finalDpId, confTx)), LOG,
1710                 "Error setting up ELAN BGs");
1711             // install L2gwDevices local macs in dpn.
1712             elanL2GatewayUtils.installL2gwDeviceMacsInDpn(dpId, externalNodeId, elanInfo, intrf.getName());
1713             // Install dpn macs on external device
1714             installDpnMacsInL2gwDevice(elanName, new HashSet<>(dpnInterfaces.getInterfaces()), dpId,
1715                     externalNodeId);
1716         }
1717         LOG.info("Handled ExternalTunnelStateEvent for {}", externalTunnel);
1718     }
1719
1720     /**
1721      * Installs dpn macs in external device. first it checks if the physical
1722      * locator towards this dpn tep is present or not if the physical locator is
1723      * present go ahead and add the ucast macs otherwise update the mcast mac
1724      * entry to include this dpn tep ip and schedule the job to put ucast macs
1725      * once the physical locator is programmed in device
1726      *
1727      * @param elanName
1728      *            the elan name
1729      * @param lstElanInterfaceNames
1730      *            the lst Elan interface names
1731      * @param dpnId
1732      *            the dpn id
1733      * @param externalNodeId
1734      *            the external node id
1735      */
1736     private void installDpnMacsInL2gwDevice(String elanName, Set<String> lstElanInterfaceNames, Uint64 dpnId,
1737             NodeId externalNodeId) {
1738         L2GatewayDevice elanL2GwDevice = ElanL2GwCacheUtils.getL2GatewayDeviceFromCache(elanName,
1739                 externalNodeId.getValue());
1740         if (elanL2GwDevice == null) {
1741             LOG.debug("L2 gw device not found in elan cache for device name {}", externalNodeId);
1742             return;
1743         }
1744         IpAddress dpnTepIp = elanItmUtils.getSourceDpnTepIp(dpnId, externalNodeId);
1745         if (dpnTepIp == null) {
1746             LOG.warn("Could not install dpn macs in l2gw device , dpnTepIp not found dpn : {} , nodeid : {}", dpnId,
1747                     externalNodeId);
1748             return;
1749         }
1750
1751         String logicalSwitchName = ElanL2GatewayUtils.getLogicalSwitchFromElan(elanName);
1752         RemoteMcastMacs remoteMcastMac = elanL2GatewayUtils.readRemoteMcastMac(externalNodeId, logicalSwitchName,
1753                 LogicalDatastoreType.OPERATIONAL);
1754         boolean phyLocAlreadyExists =
1755                 ElanL2GatewayUtils.checkIfPhyLocatorAlreadyExistsInRemoteMcastEntry(externalNodeId, remoteMcastMac,
1756                 dpnTepIp);
1757         LOG.debug("phyLocAlreadyExists = {} for locator [{}] in remote mcast entry for elan [{}], nodeId [{}]",
1758                 phyLocAlreadyExists, dpnTepIp.stringValue(), elanName, externalNodeId.getValue());
1759         List<PhysAddress> staticMacs = elanL2GatewayUtils.getElanDpnMacsFromInterfaces(lstElanInterfaceNames);
1760
1761         if (phyLocAlreadyExists) {
1762             elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1763             return;
1764         }
1765         elanL2GatewayMulticastUtils.scheduleMcastMacUpdateJob(elanName, elanL2GwDevice);
1766         elanL2GatewayUtils.scheduleAddDpnMacsInExtDevice(elanName, dpnId, staticMacs, elanL2GwDevice);
1767     }
1768
1769     /**
1770      * Validate external tunnel state event.
1771      *
1772      * @param externalTunnel
1773      *            the external tunnel
1774      * @param intrf
1775      *            the intrf
1776      * @return true, if successful
1777      */
1778     private boolean validateExternalTunnelStateEvent(ExternalTunnel externalTunnel, Interface intrf) {
1779         if (intrf.getOperStatus() == Interface.OperStatus.Up) {
1780             String srcDevice = externalTunnel.getDestinationDevice();
1781             String destDevice = externalTunnel.getSourceDevice();
1782             ExternalTunnel otherEndPointExtTunnel = elanUtils.getExternalTunnel(srcDevice, destDevice,
1783                     LogicalDatastoreType.CONFIGURATION);
1784             LOG.trace("Validating external tunnel state: src tunnel {}, dest tunnel {}", externalTunnel,
1785                     otherEndPointExtTunnel);
1786             if (otherEndPointExtTunnel != null) {
1787                 boolean otherEndPointInterfaceOperational = ElanUtils.isInterfaceOperational(
1788                         otherEndPointExtTunnel.getTunnelInterfaceName(), broker);
1789                 if (otherEndPointInterfaceOperational) {
1790                     return true;
1791                 } else {
1792                     LOG.debug("Other end [{}] of the external tunnel is not yet UP for {}",
1793                             otherEndPointExtTunnel.getTunnelInterfaceName(), externalTunnel);
1794                 }
1795             }
1796         }
1797         return false;
1798     }
1799
1800     private static List<MatchInfo> getMatchesForFilterEqualsLPortTag(int lportTag) {
1801         List<MatchInfo> mkMatches = new ArrayList<>();
1802         // Matching metadata
1803         mkMatches.add(
1804                 new MatchMetadata(MetaDataUtil.getLportTagMetaData(lportTag), MetaDataUtil.METADATA_MASK_LPORT_TAG));
1805         mkMatches.add(new MatchTunnelId(Uint64.valueOf(lportTag)));
1806         return mkMatches;
1807     }
1808
1809     @Override
1810     protected ElanInterfaceManager getDataTreeChangeListener() {
1811         return this;
1812     }
1813
1814     public void handleExternalInterfaceEvent(ElanInstance elanInstance, DpnInterfaces dpnInterfaces,
1815                                              Uint64 dpId) {
1816         LOG.debug("setting up remote BC group for elan {}", elanInstance.getPhysicalNetworkName());
1817         addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1818             confTx -> elanL2GatewayMulticastUtils.setupStandardElanBroadcastGroups(elanInstance, dpnInterfaces, dpId,
1819                 confTx)), LOG, "Error setting up remote BC group for ELAN {}", elanInstance.getPhysicalNetworkName());
1820         try {
1821             Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
1822         } catch (InterruptedException e) {
1823             LOG.warn("Error while waiting for local BC group for ELAN {} to install", elanInstance);
1824         }
1825     }
1826 }